Add guarded bot recovery and daily restore-verified realm backups

scripts/operations-agent.py

@@ -70,6 +70,18 @@ try:
  sql(f'DELETE FROM acore_auth.azerothai_snapshot_archive WHERE created_at<NOW()-INTERVAL {days} DAY LIMIT 2000')
  maintenance.append('Acknowledged transport and snapshot retention checked')
 except Exception as e:maintenance.append('Retention check needs attention: '+type(e).__name__)
+try:
+ import importlib.util
+ spec=importlib.util.spec_from_file_location('recovery_intents',repo/'scripts/recovery-intents.py')
+ recovery=importlib.util.module_from_spec(spec);spec.loader.exec_module(recovery)
+ current=json.load(urllib.request.urlopen(base+'/api/state',timeout=10))
+ count=recovery.publish_recovery_intents(current,cfg)
+ maintenance.append(f'Recovery intentions refreshed for {count} residents')
+except Exception as e:maintenance.append('Recovery check needs attention: '+type(e).__name__)
+try:
+ backup_status=json.loads((lab/'backups/latest-status.json').read_text())
+ maintenance.append(('Backup restored and verified: ' if backup_status.get('ok') else 'BACKUP NEEDS ATTENTION: ')+str(backup_status.get('at','unknown')))
+except FileNotFoundError:maintenance.append('First backup verification pending')
 service=subprocess.run(['systemctl','--user','is-active','azerothai-runtime'],capture_output=True,text=True).stdout.strip()
 send({'host':'ubuntu','at':time.time(),'cpuPercent':max(0,min(100,used)),'memoryTotalMB':mem['MemTotal'],'memoryAvailableMB':mem['MemAvailable'],'diskFreeGB':shutil.disk_usage(repo).free/1024**3,'databaseMB':bytes_in(repo/'.wrangler/state/v3/d1')/1024**2,'logsMB':sum(p.stat().st_size for p in paths if p.exists())/1024**2,'modelAvailable':model,'runtimeService':service,'maintenance':'; '.join(maintenance)[:500]})
 print(time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'telemetry and retention complete')