139 lines
8.4 KiB
Python
139 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Daily realm backup, restored in an isolated disposable MySQL container."""
|
|
import datetime
|
|
import fcntl
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import signal
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import tarfile
|
|
import time
|
|
|
|
REPO=Path(__file__).resolve().parents[1]
|
|
LAB=Path.home()/'azerothai-pc-lab'
|
|
ROOT=LAB/'backups'
|
|
DBS=('acore_auth','acore_characters','acore_playerbots')
|
|
|
|
|
|
def run(args,**kwargs):
|
|
return subprocess.run(args,check=True,**kwargs)
|
|
|
|
|
|
def main():
|
|
def interrupted(signum, frame):
|
|
raise RuntimeError("Backup interrupted")
|
|
signal.signal(signal.SIGTERM, interrupted)
|
|
os.umask(0o077)
|
|
ROOT.mkdir(parents=True,exist_ok=True)
|
|
with (ROOT/'.backup.lock').open('w') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
backup()
|
|
|
|
|
|
def backup():
|
|
if shutil.disk_usage(ROOT).free<4*1024**3:raise RuntimeError('Less than 4 GB free; backup skipped')
|
|
cfg={}
|
|
for line in (LAB/'runtime.env').read_text().splitlines():
|
|
if '=' in line and not line.startswith('#'):
|
|
k,v=line.split('=',1);cfg[k]=v.strip().strip('"').strip("'")
|
|
env=dict(os.environ,MYSQL_PWD=cfg['DOCKER_DB_ROOT_PASSWORD'])
|
|
stamp=datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
|
work=ROOT/('.pending-'+stamp);work.mkdir()
|
|
container='azerothai-restore-check-'+stamp.lower()
|
|
started=False
|
|
manifest={'at':stamp,'verified':False,'commit':subprocess.check_output(['git','-C',str(REPO),'rev-parse','HEAD'],text=True).strip()}
|
|
def sql(query,host='ac-database',credentials=env):
|
|
return subprocess.check_output(['docker','exec','-e','MYSQL_PWD',host,'mysql','-uroot','-N','-B','-e',query],env=credentials,text=True)
|
|
try:
|
|
# Snapshot each app SQLite DB using its online backup API (WAL included).
|
|
sources=list((REPO/'.wrangler/state/v3/d1').rglob('*.sqlite'))
|
|
sources=[p for p in sources if p.name!='metadata.sqlite']
|
|
if not sources:raise RuntimeError('Application database missing')
|
|
inventory={}
|
|
for i,source in enumerate(sources):
|
|
target=work/f'application-{i}.sqlite'
|
|
with sqlite3.connect('file:'+str(source)+'?mode=ro',uri=True) as live,sqlite3.connect(target) as copy:
|
|
live.backup(copy)
|
|
assert copy.execute('PRAGMA integrity_check').fetchone()[0]=='ok'
|
|
inventory[target.name]=str(source.relative_to(REPO))
|
|
manifest['sqlite']=inventory
|
|
schema_query="SELECT CONCAT(TABLE_SCHEMA,'.',TABLE_NAME) FROM information_schema.TABLES WHERE TABLE_SCHEMA IN ('acore_auth','acore_characters','acore_playerbots') AND TABLE_TYPE='BASE TABLE' ORDER BY 1"
|
|
expected=sql(schema_query).splitlines()
|
|
if not expected:raise RuntimeError('Realm database inventory empty')
|
|
with (work/'dump-error.log').open('w') as error:
|
|
proc=subprocess.Popen(['docker','exec','-e','MYSQL_PWD','ac-database','mysqldump','-uroot','--single-transaction','--quick','--no-tablespaces','--set-gtid-purged=OFF','--hex-blob','--databases',*DBS],env=env,stdout=subprocess.PIPE,stderr=error)
|
|
with gzip.open(work/'realm.sql.gz','wb') as archive:shutil.copyfileobj(proc.stdout,archive)
|
|
proc.stdout.close()
|
|
if proc.wait()!=0:raise RuntimeError('MySQL dump failed')
|
|
config_paths=[LAB/'runtime.env',REPO/'.dev.vars',LAB/'azerothcore/docker-compose.yml',LAB/'azerothcore/docker-compose.override.yml',LAB/'azerothcore/azerothai-lua']+list((Path.home()/'.config/systemd/user').glob('azerothai-*'))
|
|
with tarfile.open(work/'configuration.tar.gz','w:gz') as archive:
|
|
for p in config_paths:
|
|
if p.exists():archive.add(p,arcname=str(p.relative_to(Path.home())))
|
|
run(['git','-C',str(REPO),'bundle','create',str(work/'source.bundle'),'--all'],stdout=subprocess.DEVNULL)
|
|
# No host ports, no network and no live volumes. Data lives only in this
|
|
# disposable container's own volume; remove -v cleans it after the check.
|
|
image=subprocess.check_output(['docker','inspect','ac-database','--format={{.Config.Image}}'],text=True).strip()
|
|
password=secrets.token_hex(24)
|
|
verify_env=dict(os.environ,MYSQL_ROOT_PASSWORD=password,MYSQL_PWD=password)
|
|
run(['docker','run','-d','--name',container,'--network','none','--memory','768m','--cpus','1','-e','MYSQL_ROOT_PASSWORD',image],env=verify_env,stdout=subprocess.DEVNULL)
|
|
started=True
|
|
deadline=time.monotonic()+240
|
|
while True:
|
|
result=subprocess.run(['docker','exec','-e','MYSQL_PWD',container,'mysql','-uroot','-N','-e','SELECT 1'],env=verify_env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
if result.returncode==0:break
|
|
if time.monotonic()>deadline:raise RuntimeError('Restore-check MySQL did not start')
|
|
time.sleep(3)
|
|
with gzip.open(work/'realm.sql.gz','rb') as archive, (work/'restore-error.log').open('w') as error:
|
|
restore=subprocess.Popen(['docker','exec','-i','-e','MYSQL_PWD',container,'mysql','-uroot'],env=verify_env,stdin=subprocess.PIPE,stdout=subprocess.DEVNULL,stderr=error)
|
|
try:shutil.copyfileobj(archive,restore.stdin)
|
|
finally:restore.stdin.close()
|
|
if restore.wait()!=0:raise RuntimeError('MySQL restore failed')
|
|
restored=sql(schema_query,container,verify_env).splitlines()
|
|
if restored!=expected:raise RuntimeError('Restored table inventory differs')
|
|
# Query the restored character/lifecycle tables, never the live ones.
|
|
counts=sql('SELECT COUNT(*) FROM acore_characters.characters; SELECT COUNT(*) FROM acore_auth.azerothai_lifecycle',container,verify_env).splitlines()
|
|
if len(counts)!=2 or any(int(n)==0 for n in counts):raise RuntimeError('Restored realm is empty')
|
|
manifest['restoredTables']=len(restored);manifest['restoredCharacters']=int(counts[0]);manifest['restoredLifecycleRows']=int(counts[1])
|
|
# Restore SQLite by copying the backup to a different file and opening it.
|
|
restored_state = None
|
|
for name in inventory:
|
|
restored_file=work/('restore-'+name);shutil.copy2(work/name,restored_file)
|
|
with sqlite3.connect(restored_file) as db:
|
|
assert db.execute('PRAGMA integrity_check').fetchone()[0]=='ok'
|
|
assert db.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0]>0
|
|
if db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='realm_state'").fetchone():
|
|
row=db.execute("SELECT payload FROM realm_state WHERE id='realm'").fetchone()
|
|
if row:restored_state=json.loads(row[0])
|
|
restored_file.unlink()
|
|
if not restored_state or not restored_state.get('bots'):raise RuntimeError('Restored app realm state missing')
|
|
manifest['restoredMemories']=sum(len(b.get('memories',[])) for b in restored_state['bots'])
|
|
manifest['restoredRelationships']=sum(len(b.get('relationships',[])) for b in restored_state['bots'])
|
|
with tarfile.open(work/'configuration.tar.gz') as archive:
|
|
for member in archive:
|
|
if member.isfile():
|
|
with archive.extractfile(member) as f:
|
|
while f.read(1024*1024):pass
|
|
run(['git','-C',str(REPO),'bundle','verify',str(work/'source.bundle')],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
manifest['sha256']={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in work.iterdir() if p.is_file() and not p.name.endswith('.log')}
|
|
manifest['verified']=True
|
|
(work/'manifest.json').write_text(json.dumps(manifest,indent=2))
|
|
final=ROOT/stamp;work.rename(final)
|
|
(ROOT/'latest-status.json').write_text(json.dumps({'ok':True,'at':stamp,'path':str(final),'characters':int(counts[0]),'tables':len(restored)}))
|
|
verified=sorted(p for p in ROOT.iterdir() if p.is_dir() and (p/'manifest.json').exists() and json.loads((p/'manifest.json').read_text()).get('verified'))
|
|
for old in verified[:-7]:shutil.rmtree(old)
|
|
print(json.dumps({'verified':str(final),'tables':len(restored),'characters':int(counts[0])}),flush=True)
|
|
except Exception as exc:
|
|
(ROOT/'latest-status.json').write_text(json.dumps({'ok':False,'at':stamp,'error':str(exc)}))
|
|
raise
|
|
finally:
|
|
if started:subprocess.run(['docker','rm','-f','-v',container],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
|
|
|
|
if __name__=='__main__':main()
|