Add guarded bot recovery and daily restore-verified realm backups

scripts/operations-agent.py

@@ -70,6 +70,18 @@ try:
  sql(f'DELETE FROM acore_auth.azerothai_snapshot_archive WHERE created_at<NOW()-INTERVAL {days} DAY LIMIT 2000')
  maintenance.append('Acknowledged transport and snapshot retention checked')
 except Exception as e:maintenance.append('Retention check needs attention: '+type(e).__name__)
+try:
+ import importlib.util
+ spec=importlib.util.spec_from_file_location('recovery_intents',repo/'scripts/recovery-intents.py')
+ recovery=importlib.util.module_from_spec(spec);spec.loader.exec_module(recovery)
+ current=json.load(urllib.request.urlopen(base+'/api/state',timeout=10))
+ count=recovery.publish_recovery_intents(current,cfg)
+ maintenance.append(f'Recovery intentions refreshed for {count} residents')
+except Exception as e:maintenance.append('Recovery check needs attention: '+type(e).__name__)
+try:
+ backup_status=json.loads((lab/'backups/latest-status.json').read_text())
+ maintenance.append(('Backup restored and verified: ' if backup_status.get('ok') else 'BACKUP NEEDS ATTENTION: ')+str(backup_status.get('at','unknown')))
+except FileNotFoundError:maintenance.append('First backup verification pending')
 service=subprocess.run(['systemctl','--user','is-active','azerothai-runtime'],capture_output=True,text=True).stdout.strip()
 send({'host':'ubuntu','at':time.time(),'cpuPercent':max(0,min(100,used)),'memoryTotalMB':mem['MemTotal'],'memoryAvailableMB':mem['MemAvailable'],'diskFreeGB':shutil.disk_usage(repo).free/1024**3,'databaseMB':bytes_in(repo/'.wrangler/state/v3/d1')/1024**2,'logsMB':sum(p.stat().st_size for p in paths if p.exists())/1024**2,'modelAvailable':model,'runtimeService':service,'maintenance':'; '.join(maintenance)[:500]})
 print(time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'telemetry and retention complete')

scripts/realm-backup.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Daily realm backup, restored in an isolated disposable MySQL container."""
+import datetime
+import fcntl
+import gzip
+import hashlib
+import json
+import os
+from pathlib import Path
+import secrets
+import signal
+import shutil
+import sqlite3
+import subprocess
+import tarfile
+import time
+
+REPO=Path(__file__).resolve().parents[1]
+LAB=Path.home()/'azerothai-pc-lab'
+ROOT=LAB/'backups'
+DBS=('acore_auth','acore_characters','acore_playerbots')
+
+
+def run(args,**kwargs):
+    return subprocess.run(args,check=True,**kwargs)
+
+
+def main():
+    def interrupted(signum, frame):
+        raise RuntimeError("Backup interrupted")
+    signal.signal(signal.SIGTERM, interrupted)
+    os.umask(0o077)
+    ROOT.mkdir(parents=True,exist_ok=True)
+    with (ROOT/'.backup.lock').open('w') as lock:
+        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        backup()
+
+
+def backup():
+    if shutil.disk_usage(ROOT).free<4*1024**3:raise RuntimeError('Less than 4 GB free; backup skipped')
+    cfg={}
+    for line in (LAB/'runtime.env').read_text().splitlines():
+        if '=' in line and not line.startswith('#'):
+            k,v=line.split('=',1);cfg[k]=v.strip().strip('"').strip("'")
+    env=dict(os.environ,MYSQL_PWD=cfg['DOCKER_DB_ROOT_PASSWORD'])
+    stamp=datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ')
+    work=ROOT/('.pending-'+stamp);work.mkdir()
+    container='azerothai-restore-check-'+stamp.lower()
+    started=False
+    manifest={'at':stamp,'verified':False,'commit':subprocess.check_output(['git','-C',str(REPO),'rev-parse','HEAD'],text=True).strip()}
+    def sql(query,host='ac-database',credentials=env):
+        return subprocess.check_output(['docker','exec','-e','MYSQL_PWD',host,'mysql','-uroot','-N','-B','-e',query],env=credentials,text=True)
+    try:
+        # Snapshot each app SQLite DB using its online backup API (WAL included).
+        sources=list((REPO/'.wrangler/state/v3/d1').rglob('*.sqlite'))
+        sources=[p for p in sources if p.name!='metadata.sqlite']
+        if not sources:raise RuntimeError('Application database missing')
+        inventory={}
+        for i,source in enumerate(sources):
+            target=work/f'application-{i}.sqlite'
+            with sqlite3.connect('file:'+str(source)+'?mode=ro',uri=True) as live,sqlite3.connect(target) as copy:
+                live.backup(copy)
+                assert copy.execute('PRAGMA integrity_check').fetchone()[0]=='ok'
+                inventory[target.name]=str(source.relative_to(REPO))
+        manifest['sqlite']=inventory
+        schema_query="SELECT CONCAT(TABLE_SCHEMA,'.',TABLE_NAME) FROM information_schema.TABLES WHERE TABLE_SCHEMA IN ('acore_auth','acore_characters','acore_playerbots') AND TABLE_TYPE='BASE TABLE' ORDER BY 1"
+        expected=sql(schema_query).splitlines()
+        if not expected:raise RuntimeError('Realm database inventory empty')
+        with (work/'dump-error.log').open('w') as error:
+            proc=subprocess.Popen(['docker','exec','-e','MYSQL_PWD','ac-database','mysqldump','-uroot','--single-transaction','--quick','--no-tablespaces','--set-gtid-purged=OFF','--hex-blob','--databases',*DBS],env=env,stdout=subprocess.PIPE,stderr=error)
+            with gzip.open(work/'realm.sql.gz','wb') as archive:shutil.copyfileobj(proc.stdout,archive)
+            proc.stdout.close()
+            if proc.wait()!=0:raise RuntimeError('MySQL dump failed')
+        config_paths=[LAB/'runtime.env',REPO/'.dev.vars',LAB/'azerothcore/docker-compose.yml',LAB/'azerothcore/docker-compose.override.yml',LAB/'azerothcore/azerothai-lua']+list((Path.home()/'.config/systemd/user').glob('azerothai-*'))
+        with tarfile.open(work/'configuration.tar.gz','w:gz') as archive:
+            for p in config_paths:
+                if p.exists():archive.add(p,arcname=str(p.relative_to(Path.home())))
+        run(['git','-C',str(REPO),'bundle','create',str(work/'source.bundle'),'--all'],stdout=subprocess.DEVNULL)
+        # No host ports, no network and no live volumes. Data lives only in this
+        # disposable container's own volume; remove -v cleans it after the check.
+        image=subprocess.check_output(['docker','inspect','ac-database','--format={{.Config.Image}}'],text=True).strip()
+        password=secrets.token_hex(24)
+        verify_env=dict(os.environ,MYSQL_ROOT_PASSWORD=password,MYSQL_PWD=password)
+        run(['docker','run','-d','--name',container,'--network','none','--memory','768m','--cpus','1','-e','MYSQL_ROOT_PASSWORD',image],env=verify_env,stdout=subprocess.DEVNULL)
+        started=True
+        deadline=time.monotonic()+240
+        while True:
+            result=subprocess.run(['docker','exec','-e','MYSQL_PWD',container,'mysql','-uroot','-N','-e','SELECT 1'],env=verify_env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+            if result.returncode==0:break
+            if time.monotonic()>deadline:raise RuntimeError('Restore-check MySQL did not start')
+            time.sleep(3)
+        with gzip.open(work/'realm.sql.gz','rb') as archive, (work/'restore-error.log').open('w') as error:
+            restore=subprocess.Popen(['docker','exec','-i','-e','MYSQL_PWD',container,'mysql','-uroot'],env=verify_env,stdin=subprocess.PIPE,stdout=subprocess.DEVNULL,stderr=error)
+            try:shutil.copyfileobj(archive,restore.stdin)
+            finally:restore.stdin.close()
+            if restore.wait()!=0:raise RuntimeError('MySQL restore failed')
+        restored=sql(schema_query,container,verify_env).splitlines()
+        if restored!=expected:raise RuntimeError('Restored table inventory differs')
+        # Query the restored character/lifecycle tables, never the live ones.
+        counts=sql('SELECT COUNT(*) FROM acore_characters.characters; SELECT COUNT(*) FROM acore_auth.azerothai_lifecycle',container,verify_env).splitlines()
+        if len(counts)!=2 or any(int(n)==0 for n in counts):raise RuntimeError('Restored realm is empty')
+        manifest['restoredTables']=len(restored);manifest['restoredCharacters']=int(counts[0]);manifest['restoredLifecycleRows']=int(counts[1])
+        # Restore SQLite by copying the backup to a different file and opening it.
+        restored_state = None
+        for name in inventory:
+            restored_file=work/('restore-'+name);shutil.copy2(work/name,restored_file)
+            with sqlite3.connect(restored_file) as db:
+                assert db.execute('PRAGMA integrity_check').fetchone()[0]=='ok'
+                assert db.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0]>0
+                if db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='realm_state'").fetchone():
+                    row=db.execute("SELECT payload FROM realm_state WHERE id='realm'").fetchone()
+                    if row:restored_state=json.loads(row[0])
+            restored_file.unlink()
+        if not restored_state or not restored_state.get('bots'):raise RuntimeError('Restored app realm state missing')
+        manifest['restoredMemories']=sum(len(b.get('memories',[])) for b in restored_state['bots'])
+        manifest['restoredRelationships']=sum(len(b.get('relationships',[])) for b in restored_state['bots'])
+        with tarfile.open(work/'configuration.tar.gz') as archive:
+            for member in archive:
+                if member.isfile():
+                    with archive.extractfile(member) as f:
+                        while f.read(1024*1024):pass
+        run(['git','-C',str(REPO),'bundle','verify',str(work/'source.bundle')],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+        manifest['sha256']={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in work.iterdir() if p.is_file() and not p.name.endswith('.log')}
+        manifest['verified']=True
+        (work/'manifest.json').write_text(json.dumps(manifest,indent=2))
+        final=ROOT/stamp;work.rename(final)
+        (ROOT/'latest-status.json').write_text(json.dumps({'ok':True,'at':stamp,'path':str(final),'characters':int(counts[0]),'tables':len(restored)}))
+        verified=sorted(p for p in ROOT.iterdir() if p.is_dir() and (p/'manifest.json').exists() and json.loads((p/'manifest.json').read_text()).get('verified'))
+        for old in verified[:-7]:shutil.rmtree(old)
+        print(json.dumps({'verified':str(final),'tables':len(restored),'characters':int(counts[0])}),flush=True)
+    except Exception as exc:
+        (ROOT/'latest-status.json').write_text(json.dumps({'ok':False,'at':stamp,'error':str(exc)}))
+        raise
+    finally:
+        if started:subprocess.run(['docker','rm','-f','-v',container],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+
+
+if __name__=='__main__':main()

scripts/recovery-intents.py

@@ -0,0 +1,23 @@
+"""Publish only eligible live intentions to the native recovery guard."""
+import json
+import time
+import urllib.request
+
+
+def publish_recovery_intents(state, cfg):
+    def literal(value):
+        return '"'+''.join('\\%03d'%c for c in value.encode())+'"'
+    rows=[]
+    for bot in state.get('bots',[]):
+        identity=bot.get('autonomy') or {};world=bot.get('world') or {}
+        action=(bot.get('lastDecision') or {}).get('action')
+        if bot.get('status')!='online' or not identity.get('managed') or action not in ('quest','explore'):continue
+        if time.time()-world.get('at',0)>120:continue
+        guid=identity.get('characterGuid');account=identity.get('accountId')
+        if type(guid) is not int or type(account) is not int or guid<=0 or account<=0:continue
+        rows.append('{guid=%d,account=%d,name=%s,action=%s}'%(guid,account,literal(bot['name']),literal(action)))
+    source='if not AzerothAISetRecoveryIntents then error("Recovery guard not loaded") end; return AzerothAISetRecoveryIntents({'+','.join(rows)+'})'
+    req=urllib.request.Request('http://127.0.0.1:8787/api/ale/operator',data=json.dumps({'kind':'raw','source':source}).encode(),headers={'Content-Type':'application/json','Authorization':'Bearer '+cfg['ALE_OPERATOR_TOKEN']})
+    with urllib.request.urlopen(req,timeout=10) as r:
+        if r.status not in (200,202):raise RuntimeError('Recovery publication failed')
+    return len(rows)