josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild missing containers during maintenance recovery

Browse Source
This commit is contained in:
josh
2026-10-08 13:33:01 -04:00
parent 368e457dc6
commit b3fa486886
9 changed files with 201 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/maintenance/recovery.ts
+9 -4
View File
@@ -247,7 +247,7 @@ async function observe(server: string, operationId?: string, fresh = false): Pro
const identity = snapshot?.identity ?? monitoringIdentity(server);
const containerEvidence = { identity, observedAt: inspected.observedAt };
const state = health.state?.toLowerCase() ?? null;
if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['missing','stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
const telemetry = telemetryEvidence(server);
if (telemetry) return { identity, observedAt: telemetry.receivedAt, healthy: true, actionable: false, reason: 'Container healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
const probed = snapshot ? snapshot.game : await rconForMonitoring(server, true).catch(error => {
@@ -280,16 +280,21 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
const oldEnough = Date.parse(observation.observedAt) - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
if (problem.state === 'open' && observation.actionable && oldEnough && problem.failures >= 3 && maintenanceAllowed('health')) {
// A missing container has nothing to restart. Recreate it through
// the normal rebuild queue, with its identity and safety checks.
const action = observation.containerState === 'missing' ? 'rebuild' : 'restart';
const result = db().transaction(() => {
const next = requestWatchdogLifecycle(server, 'restart', problem.id);
if (next.ok) addAction(problem, next.deploymentId, 'restart');
const next = requestWatchdogLifecycle(server, action, problem.id);
if (next.ok) addAction(problem, next.deploymentId, action);
return next;
})();
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error);
}
}
// Missing-container recovery belongs to the health policy. Do not turn
// absent files into drift or bypass the sustained health-failure window.
// Unknown container health is an observation failure, not proof that a rebuild is safe.
if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
if (observation.containerState === 'missing' || (!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
// File trees change far less often than liveness. Persist the last successful
// scan per registration/process so health checks do not rescan every minute.
// Manual checks and new lifecycle submissions always get fresh file evidence.