#!/usr/bin/env python3 """Daily realm backup, restored in an isolated disposable MySQL container.""" import datetime import fcntl import gzip import hashlib import json import os from pathlib import Path import secrets import signal import shutil import sqlite3 import subprocess import tarfile import time REPO=Path(__file__).resolve().parents[1] LAB=Path.home()/'azerothai-pc-lab' ROOT=LAB/'backups' DBS=('acore_auth','acore_characters','acore_playerbots') def run(args,**kwargs): return subprocess.run(args,check=True,**kwargs) def main(): def interrupted(signum, frame): raise RuntimeError("Backup interrupted") signal.signal(signal.SIGTERM, interrupted) os.umask(0o077) ROOT.mkdir(parents=True,exist_ok=True) with (ROOT/'.backup.lock').open('w') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) backup() def backup(): if shutil.disk_usage(ROOT).free<4*1024**3:raise RuntimeError('Less than 4 GB free; backup skipped') cfg={} for line in (LAB/'runtime.env').read_text().splitlines(): if '=' in line and not line.startswith('#'): k,v=line.split('=',1);cfg[k]=v.strip().strip('"').strip("'") env=dict(os.environ,MYSQL_PWD=cfg['DOCKER_DB_ROOT_PASSWORD']) stamp=datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ') work=ROOT/('.pending-'+stamp);work.mkdir() container='azerothai-restore-check-'+stamp.lower() started=False manifest={'at':stamp,'verified':False,'commit':subprocess.check_output(['git','-C',str(REPO),'rev-parse','HEAD'],text=True).strip()} def sql(query,host='ac-database',credentials=env): return subprocess.check_output(['docker','exec','-e','MYSQL_PWD',host,'mysql','-uroot','-N','-B','-e',query],env=credentials,text=True) try: # Snapshot each app SQLite DB using its online backup API (WAL included). sources=list((REPO/'.wrangler/state/v3/d1').rglob('*.sqlite')) sources=[p for p in sources if p.name!='metadata.sqlite'] if not sources:raise RuntimeError('Application database missing') inventory={} for i,source in enumerate(sources): target=work/f'application-{i}.sqlite' with sqlite3.connect('file:'+str(source)+'?mode=ro',uri=True) as live,sqlite3.connect(target) as copy: live.backup(copy) assert copy.execute('PRAGMA integrity_check').fetchone()[0]=='ok' inventory[target.name]=str(source.relative_to(REPO)) manifest['sqlite']=inventory schema_query="SELECT CONCAT(TABLE_SCHEMA,'.',TABLE_NAME) FROM information_schema.TABLES WHERE TABLE_SCHEMA IN ('acore_auth','acore_characters','acore_playerbots') AND TABLE_TYPE='BASE TABLE' ORDER BY 1" expected=sql(schema_query).splitlines() if not expected:raise RuntimeError('Realm database inventory empty') with (work/'dump-error.log').open('w') as error: proc=subprocess.Popen(['docker','exec','-e','MYSQL_PWD','ac-database','mysqldump','-uroot','--single-transaction','--quick','--no-tablespaces','--set-gtid-purged=OFF','--hex-blob','--databases',*DBS],env=env,stdout=subprocess.PIPE,stderr=error) with gzip.open(work/'realm.sql.gz','wb') as archive:shutil.copyfileobj(proc.stdout,archive) proc.stdout.close() if proc.wait()!=0:raise RuntimeError('MySQL dump failed') config_paths=[LAB/'runtime.env',REPO/'.dev.vars',LAB/'azerothcore/docker-compose.yml',LAB/'azerothcore/docker-compose.override.yml',LAB/'azerothcore/azerothai-lua']+list((Path.home()/'.config/systemd/user').glob('azerothai-*')) with tarfile.open(work/'configuration.tar.gz','w:gz') as archive: for p in config_paths: if p.exists():archive.add(p,arcname=str(p.relative_to(Path.home()))) run(['git','-C',str(REPO),'bundle','create',str(work/'source.bundle'),'--all'],stdout=subprocess.DEVNULL) # No host ports, no network and no live volumes. Data lives only in this # disposable container's own volume; remove -v cleans it after the check. image=subprocess.check_output(['docker','inspect','ac-database','--format={{.Config.Image}}'],text=True).strip() password=secrets.token_hex(24) verify_env=dict(os.environ,MYSQL_ROOT_PASSWORD=password,MYSQL_PWD=password) run(['docker','run','-d','--name',container,'--network','none','--memory','768m','--cpus','1','-e','MYSQL_ROOT_PASSWORD',image],env=verify_env,stdout=subprocess.DEVNULL) started=True deadline=time.monotonic()+240 while True: result=subprocess.run(['docker','exec','-e','MYSQL_PWD',container,'mysql','-uroot','-N','-e','SELECT 1'],env=verify_env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) if result.returncode==0:break if time.monotonic()>deadline:raise RuntimeError('Restore-check MySQL did not start') time.sleep(3) with gzip.open(work/'realm.sql.gz','rb') as archive, (work/'restore-error.log').open('w') as error: restore=subprocess.Popen(['docker','exec','-i','-e','MYSQL_PWD',container,'mysql','-uroot'],env=verify_env,stdin=subprocess.PIPE,stdout=subprocess.DEVNULL,stderr=error) try:shutil.copyfileobj(archive,restore.stdin) finally:restore.stdin.close() if restore.wait()!=0:raise RuntimeError('MySQL restore failed') restored=sql(schema_query,container,verify_env).splitlines() if restored!=expected:raise RuntimeError('Restored table inventory differs') # Query the restored character/lifecycle tables, never the live ones. counts=sql('SELECT COUNT(*) FROM acore_characters.characters; SELECT COUNT(*) FROM acore_auth.azerothai_lifecycle',container,verify_env).splitlines() if len(counts)!=2 or any(int(n)==0 for n in counts):raise RuntimeError('Restored realm is empty') manifest['restoredTables']=len(restored);manifest['restoredCharacters']=int(counts[0]);manifest['restoredLifecycleRows']=int(counts[1]) # Restore SQLite by copying the backup to a different file and opening it. restored_state = None for name in inventory: restored_file=work/('restore-'+name);shutil.copy2(work/name,restored_file) with sqlite3.connect(restored_file) as db: assert db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' assert db.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0]>0 if db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='realm_state'").fetchone(): row=db.execute("SELECT payload FROM realm_state WHERE id='realm'").fetchone() if row: restored_state=json.loads(row[0]) if restored_state.get('chunkToken'): chunks=db.execute('SELECT payload FROM realm_state_chunks WHERE token=? ORDER BY part',(restored_state['chunkToken'],)).fetchall() assert len(chunks)==restored_state['parts'], 'Incomplete restored state' restored_state=json.loads(''.join(r[0] for r in chunks)) restored_file.unlink() if not restored_state or not restored_state.get('bots'):raise RuntimeError('Restored app realm state missing') manifest['restoredMemories']=sum(len(b.get('memories',[])) for b in restored_state['bots']) manifest['restoredRelationships']=sum(len(b.get('relationships',[])) for b in restored_state['bots']) with tarfile.open(work/'configuration.tar.gz') as archive: for member in archive: if member.isfile(): with archive.extractfile(member) as f: while f.read(1024*1024):pass run(['git','-C',str(REPO),'bundle','verify',str(work/'source.bundle')],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) manifest['sha256']={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in work.iterdir() if p.is_file() and not p.name.endswith('.log')} manifest['verified']=True (work/'manifest.json').write_text(json.dumps(manifest,indent=2)) final=ROOT/stamp;work.rename(final) (ROOT/'latest-status.json').write_text(json.dumps({'ok':True,'at':stamp,'path':str(final),'characters':int(counts[0]),'tables':len(restored)})) verified=sorted(p for p in ROOT.iterdir() if p.is_dir() and (p/'manifest.json').exists() and json.loads((p/'manifest.json').read_text()).get('verified')) for old in verified[:-7]:shutil.rmtree(old) print(json.dumps({'verified':str(final),'tables':len(restored),'characters':int(counts[0])}),flush=True) except Exception as exc: (ROOT/'latest-status.json').write_text(json.dumps({'ok':False,'at':stamp,'error':str(exc)})) raise finally: if started:subprocess.run(['docker','rm','-f','-v',container],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) if __name__=='__main__':main()