diff --git a/ale/azerothai_recovery.lua b/ale/azerothai_recovery.lua new file mode 100644 index 0000000..8b28f55 --- /dev/null +++ b/ale/azerothai_recovery.lua @@ -0,0 +1,69 @@ +-- Conservative recovery for managed residents with fresh quest/explore intent. +-- Native position/progress is sampled independently of dashboard summaries. +local intents, observations = {}, {} +local LIMIT = 600 +local COOLDOWN = 1800 +local function sqltext(s) + return "CONVERT(UNHEX('"..string.gsub(s,".",function(c)return string.format('%02X',string.byte(c))end).."') USING utf8mb4)" +end +AuthDBExecute([[CREATE TABLE IF NOT EXISTS azerothai_recovery_log ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,guid INT UNSIGNED NOT NULL, + name VARCHAR(64) NOT NULL,at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + detail VARCHAR(255) NOT NULL,INDEX(guid,at))]]) +function AzerothAISetRecoveryIntents(rows) + local nextIntents={} + for _,r in ipairs(rows) do + if type(r.guid)=='number' and type(r.account)=='number' and type(r.name)=='string' + and (r.action=='quest' or r.action=='explore') then nextIntents[r.guid]=r;r.at=os.time() end + end + intents=nextIntents + return 'Recovery intentions refreshed' +end +local function safe(p,a) + if p:IsDead() or p:IsInCombat() or p:IsCasting() or p:IsTaxi() or a:HasGameClientMaster() + or p:IsRooted() or a:IsMovementImpaired() then return false end + local g=p:GetGroup() + if g then + if #g:GetMembers()~=g:GetMembersCount() then return false end + for _,m in pairs(g:GetMembers()) do if not m:IsPlayerbot() then return false end end + end + return true +end +local function sample() + local now=os.time() + for guid,r in pairs(intents) do + local p=GetPlayerByName(r.name) + if now-r.at>120 or not p or not p:IsPlayerbot() or p:GetGUIDLow()~=guid or p:GetAccountId()~=r.account then + observations[guid]=nil + else + local a=p:GetBotAI() + if not a or not safe(p,a) then observations[guid]=nil + else + local x,y,z,map=p:GetX(),p:GetY(),p:GetZ(),p:GetMapId() + local level,hp=p:GetLevel(),p:GetHealth() + local old=observations[guid] + if not old or old.map~=map or old.level~=level or old.hp~=hp or (x-old.x)^2+(y-old.y)^2+(z-old.z)^2>25 then + observations[guid]={x=x,y=y,z=z,map=map,level=level,hp=hp,since=now} + elseif now-old.since>=LIMIT then + -- Persistent cooldown survives ALE reloads; exact identity must still be active. + local q=AuthDBQuery('SELECT guid FROM azerothai_lifecycle WHERE guid='..guid..' AND account_id='..r.account..' AND BINARY name='..sqltext(r.name).." AND state='active'") + local recent=AuthDBQuery('SELECT id FROM azerothai_recovery_log WHERE guid='..guid..' AND at>NOW()-INTERVAL '..COOLDOWN..' SECOND LIMIT 1') + if q and not recent then + local staleFlight=p:GetMovementType()==7 and not p:IsTaxi() + if staleFlight then p:MoveExpire();p:MoveClear() end + a:Reset(true) + local moved=a:DoSpecificAction('move random',true) + local detail=(staleFlight and 'Cleared stale flight; ' or '')..'AI replanned; walking request='..tostring(moved) + AuthDBExecute('INSERT INTO azerothai_recovery_log(guid,name,detail) VALUES ('..guid..','..sqltext(r.name)..','..sqltext(detail)..')') + PrintInfo('[AzerothAI recovery] '..r.name..': '..detail) + end + old.since=now + end + end + end + end + -- Forget residents whose intent changed to resting/socializing or who went offline. + for guid in pairs(observations) do if not intents[guid] then observations[guid]=nil end end +end +CreateLuaEvent(function()local ok,err=pcall(sample);if not ok then PrintError('[AzerothAI recovery] '..tostring(err)) end end,30000,0) +PrintInfo('[AzerothAI] Conservative stuck recovery enabled: 10 minutes idle, 30 minute cooldown') diff --git a/deploy/playable/install-safety-services.py b/deploy/playable/install-safety-services.py new file mode 100644 index 0000000..6ada01e --- /dev/null +++ b/deploy/playable/install-safety-services.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +from pathlib import Path +import shutil,subprocess +repo=Path(__file__).resolve().parents[2] +lab=Path.home()/'azerothai-pc-lab' +shutil.copy2(repo/'ale/azerothai_recovery.lua',lab/'azerothcore/azerothai-lua/azerothai_recovery.lua') +units=Path.home()/'.config/systemd/user';units.mkdir(parents=True,exist_ok=True) +python=shutil.which('python3');assert python +(units/'azerothai-backup.service').write_text(f'''[Unit] +Description=AzerothAI backup and isolated restore verification +[Service] +Type=oneshot +WorkingDirectory={repo} +ExecStart=/usr/bin/sg docker -c "{python} {repo}/scripts/realm-backup.py" +TimeoutStartSec=30min +Nice=10 +UMask=0077 +''') +(units/'azerothai-backup.timer').write_text('''[Unit] +Description=AzerothAI daily verified backup +[Timer] +OnCalendar=*-*-* 06:00:00 UTC +Persistent=true +RandomizedDelaySec=300 +[Install] +WantedBy=timers.target +''') +subprocess.run(['systemctl','--user','daemon-reload'],check=True) +subprocess.run(['systemctl','--user','enable','--now','azerothai-backup.timer'],check=True) +print('Daily backup scheduled for 06:00 UTC; reload ALE to enable native recovery.') diff --git a/docs/recovery-and-backups.md b/docs/recovery-and-backups.md new file mode 100644 index 0000000..586916a --- /dev/null +++ b/docs/recovery-and-backups.md @@ -0,0 +1,82 @@ +# Recovery and verified backups + +## Bot recovery + +The minute operations job supplies fresh, managed quest/explore intentions to +`ale/azerothai_recovery.lua`. A native 30-second sampler independently observes +position, map, level and health. Ten minutes without moving over five yards or +making observable progress makes a resident eligible for recovery. Combat, +casting, taxi travel, death, roots, movement impairment, a human master, or a +party containing humans/offline members reset that observation. Rest, trade and +social intentions are excluded. Inputs expire after two minutes if telemetry or +publication stops. Every action checks the exact active native identity. + +Recovery clears a stale flight movement controller only when the character is +not actually on a taxi, then resets AI caches and asks native AI for a short +collision/path-checked walk. It never teleports, revives, changes health, resets +levels, deletes memories, or increases the population cap. A persistent SQL log +`acore_auth.azerothai_recovery_log` records attempts and enforces a thirty-minute +per-resident cooldown. A walking request succeeding is not proof the bot remains +unstuck: subsequent sampling observes actual progress. Repeated failures remain +visible in the existing System Health stalled-bot list. + +## Backups + +`azerothai-backup.timer` runs daily at 06:00 UTC, with up to five minutes of +jitter, and catches missed runs after boot. The last seven **verified** snapshots +are retained under `~/azerothai-pc-lab/backups`. Backup folders and credentials +are accessible only to the realm owner. Check `latest-status.json` or System +Health maintenance status; failures also appear in the user systemd journal. + +Each snapshot includes: + +- Transactional MySQL dumps of auth, characters, and playerbots databases, + including lifecycle/recovery ledgers and transport data. +- Online SQLite snapshots including bot memories, relationships and guild goals. +- Runtime configuration, deployed Lua, compose configuration and AzerothAI units. +- A Git bundle, commit identifier, file hashes, and verification manifest. + +Each run imports the SQL into a disposable MySQL container using the same image, +with no network, no published ports and no live database volumes. It checks the +restored table inventory and character/lifecycle data. SQLite backups are copied, +reopened and integrity-checked; the restored realm state, memories and bonds are +read. Configuration archives and the Git bundle are checked as well. The test +container and its volume are removed afterward. A failed verification never +replaces a verified snapshot or triggers pruning of good backups. + +These are live, independently consistent database snapshots, not one atomic +cross-database timestamp. Static AzerothCore world data, client assets and model +weights are not duplicated; reinstall the matching versions when rebuilding. +Snapshots currently live inside the Ubuntu VM on the same physical PC: they +protect against application mistakes and corruption, not loss of that PC/disk. + +## Operation and recovery + +Install as the realm owner: + +```sh +python3 deploy/playable/install-safety-services.py +``` + +Reload ALE after installing the guard. Existing operations and user lingering +services provide the intention updates and reboot startup. Trigger a backup: + +```sh +systemctl --user start azerothai-backup +``` + +Read verification results: + +```sh +cat ~/azerothai-pc-lab/backups/latest-status.json +journalctl --user -u azerothai-backup --no-pager -n 30 +``` + +For disaster recovery, use a fresh isolated realm first: verify manifest hashes, +restore the source bundle at the recorded commit, recreate matching database +containers, import `realm.sql.gz`, restore each application SQLite file to the +path recorded in the manifest, and restore configuration with owner-only access. +Stop the app, relay/autonomy workers and world/auth servers before replacing any +live databases. Keep the previous live data until the recovered realm has been +checked. Never import a backup into an actively running game. The automatic +restore check does not overwrite the live realm. diff --git a/scripts/operations-agent.py b/scripts/operations-agent.py index 0c5d923..9914b4c 100644 --- a/scripts/operations-agent.py +++ b/scripts/operations-agent.py @@ -70,6 +70,18 @@ try: sql(f'DELETE FROM acore_auth.azerothai_snapshot_archive WHERE created_atdeadline:raise RuntimeError('Restore-check MySQL did not start') + time.sleep(3) + with gzip.open(work/'realm.sql.gz','rb') as archive, (work/'restore-error.log').open('w') as error: + restore=subprocess.Popen(['docker','exec','-i','-e','MYSQL_PWD',container,'mysql','-uroot'],env=verify_env,stdin=subprocess.PIPE,stdout=subprocess.DEVNULL,stderr=error) + try:shutil.copyfileobj(archive,restore.stdin) + finally:restore.stdin.close() + if restore.wait()!=0:raise RuntimeError('MySQL restore failed') + restored=sql(schema_query,container,verify_env).splitlines() + if restored!=expected:raise RuntimeError('Restored table inventory differs') + # Query the restored character/lifecycle tables, never the live ones. + counts=sql('SELECT COUNT(*) FROM acore_characters.characters; SELECT COUNT(*) FROM acore_auth.azerothai_lifecycle',container,verify_env).splitlines() + if len(counts)!=2 or any(int(n)==0 for n in counts):raise RuntimeError('Restored realm is empty') + manifest['restoredTables']=len(restored);manifest['restoredCharacters']=int(counts[0]);manifest['restoredLifecycleRows']=int(counts[1]) + # Restore SQLite by copying the backup to a different file and opening it. + restored_state = None + for name in inventory: + restored_file=work/('restore-'+name);shutil.copy2(work/name,restored_file) + with sqlite3.connect(restored_file) as db: + assert db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' + assert db.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0]>0 + if db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='realm_state'").fetchone(): + row=db.execute("SELECT payload FROM realm_state WHERE id='realm'").fetchone() + if row:restored_state=json.loads(row[0]) + restored_file.unlink() + if not restored_state or not restored_state.get('bots'):raise RuntimeError('Restored app realm state missing') + manifest['restoredMemories']=sum(len(b.get('memories',[])) for b in restored_state['bots']) + manifest['restoredRelationships']=sum(len(b.get('relationships',[])) for b in restored_state['bots']) + with tarfile.open(work/'configuration.tar.gz') as archive: + for member in archive: + if member.isfile(): + with archive.extractfile(member) as f: + while f.read(1024*1024):pass + run(['git','-C',str(REPO),'bundle','verify',str(work/'source.bundle')],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + manifest['sha256']={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in work.iterdir() if p.is_file() and not p.name.endswith('.log')} + manifest['verified']=True + (work/'manifest.json').write_text(json.dumps(manifest,indent=2)) + final=ROOT/stamp;work.rename(final) + (ROOT/'latest-status.json').write_text(json.dumps({'ok':True,'at':stamp,'path':str(final),'characters':int(counts[0]),'tables':len(restored)})) + verified=sorted(p for p in ROOT.iterdir() if p.is_dir() and (p/'manifest.json').exists() and json.loads((p/'manifest.json').read_text()).get('verified')) + for old in verified[:-7]:shutil.rmtree(old) + print(json.dumps({'verified':str(final),'tables':len(restored),'characters':int(counts[0])}),flush=True) + except Exception as exc: + (ROOT/'latest-status.json').write_text(json.dumps({'ok':False,'at':stamp,'error':str(exc)})) + raise + finally: + if started:subprocess.run(['docker','rm','-f','-v',container],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + + +if __name__=='__main__':main() diff --git a/scripts/recovery-intents.py b/scripts/recovery-intents.py new file mode 100644 index 0000000..321a4cf --- /dev/null +++ b/scripts/recovery-intents.py @@ -0,0 +1,23 @@ +"""Publish only eligible live intentions to the native recovery guard.""" +import json +import time +import urllib.request + + +def publish_recovery_intents(state, cfg): + def literal(value): + return '"'+''.join('\\%03d'%c for c in value.encode())+'"' + rows=[] + for bot in state.get('bots',[]): + identity=bot.get('autonomy') or {};world=bot.get('world') or {} + action=(bot.get('lastDecision') or {}).get('action') + if bot.get('status')!='online' or not identity.get('managed') or action not in ('quest','explore'):continue + if time.time()-world.get('at',0)>120:continue + guid=identity.get('characterGuid');account=identity.get('accountId') + if type(guid) is not int or type(account) is not int or guid<=0 or account<=0:continue + rows.append('{guid=%d,account=%d,name=%s,action=%s}'%(guid,account,literal(bot['name']),literal(action))) + source='if not AzerothAISetRecoveryIntents then error("Recovery guard not loaded") end; return AzerothAISetRecoveryIntents({'+','.join(rows)+'})' + req=urllib.request.Request('http://127.0.0.1:8787/api/ale/operator',data=json.dumps({'kind':'raw','source':source}).encode(),headers={'Content-Type':'application/json','Authorization':'Bearer '+cfg['ALE_OPERATOR_TOKEN']}) + with urllib.request.urlopen(req,timeout=10) as r: + if r.status not in (200,202):raise RuntimeError('Recovery publication failed') + return len(rows) diff --git a/tests/recovery-lua.test.py b/tests/recovery-lua.test.py new file mode 100644 index 0000000..7de045c --- /dev/null +++ b/tests/recovery-lua.test.py @@ -0,0 +1,57 @@ +from pathlib import Path +from lupa import LuaRuntime +code=(Path(__file__).resolve().parents[1]/'ale/azerothai_recovery.lua').read_text() +setup=''' +now=1000;os.time=function()return now end;attempts=0;expired=0;last=nil;active=true;player={x=0};ai={};captured={} +function AuthDBExecute(sql) if string.find(sql,'INSERT INTO',1,true) then last=now end end +function AuthDBQuery(sql) if string.find(sql,'recovery_log',1,true) then return last and now-last<1800 and {} or nil end;return active and {} or nil end +function GetPlayerByName(n) return n=='Test' and player or nil end +function CreateLuaEvent(fn) tick=fn end +function PrintInfo(s) end +function PrintError(s) error(s) end +function player:GetGUIDLow()return 9 end +function player:GetAccountId()return 2 end +function player:IsPlayerbot()return true end +function player:GetBotAI()return ai end +function player:IsDead()return self.dead end +function player:IsInCombat()return self.combat end +function player:IsCasting()return self.casting end +function player:IsTaxi()return self.taxi end +function player:IsRooted()return self.root end +function player:GetGroup()return self.group end +function player:GetX()return self.x end +function player:GetY()return 0 end +function player:GetZ()return 0 end +function player:GetMapId()return 0 end +function player:GetLevel()return 8 end +function player:GetHealth()return self.hp or 100 end +function player:GetMovementType()return self.motion or 0 end +function player:MoveExpire()expired=expired+1 end +function player:MoveClear()end +function ai:HasGameClientMaster()return player.master end +function ai:IsMovementImpaired()return player.impaired end +function ai:Reset(full)attempts=attempts+1 end +function ai:DoSpecificAction(action,silent)assert(action=='move random');return true end +function step(seconds,action) + now=now+seconds + AzerothAISetRecoveryIntents({{guid=9,account=2,name='Test',action=action or 'quest'}}) + tick() +end +''' +def new(): + l=LuaRuntime();l.execute(setup);l.execute(code);return l +l=new();l.execute('step(0);step(599);assert(attempts==0);step(1);assert(attempts==1);step(600);assert(attempts==1);step(1200);assert(attempts==2)') +print('Threshold and persistent cooldown passed') +for field in ['dead','combat','casting','taxi','root','master','impaired']: + l=new();l.execute(f'player.{field}=true;step(0);step(601);assert(attempts==0)') +print('Death, combat, cast, taxi, root, master and impaired exclusions passed') +for action in ['rest','socialize','trade']: + l=new();l.execute(f'step(0,"{action}");step(601,"{action}");assert(attempts==0)') +l=new();l.execute('step(0);player.x=8;step(599);step(2);assert(attempts==0)') +l=new();l.execute('step(0);player.hp=99;step(599);step(2);assert(attempts==0)') +l=new();l.execute('step(0);active=false;step(601);assert(attempts==0)') +l=new();l.execute('step(0);now=now+601;tick();assert(attempts==0)') +print('Intent, movement, health progress, active identity and stale input guards passed') +l=new();l.execute('player.group={GetMembers=function()return {{IsPlayerbot=function()return false end}} end,GetMembersCount=function()return 1 end};step(0);step(601);assert(attempts==0)') +l=new();l.execute('player.motion=7;step(0);step(601);assert(attempts==1 and expired==1)') +print('Human group exclusion and stale flight recovery passed')