josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild missing containers during maintenance recovery

Browse Source
This commit is contained in:
josh
2026-10-08 13:33:01 -04:00
parent 368e457dc6
commit b3fa486886
9 changed files with 201 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/maintenance/recovery.ts
+9 -4
View File
@@ -247,7 +247,7 @@ async function observe(server: string, operationId?: string, fresh = false): Pro
const identity = snapshot?.identity ?? monitoringIdentity(server);
const containerEvidence = { identity, observedAt: inspected.observedAt };
const state = health.state?.toLowerCase() ?? null;
if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['missing','stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
const telemetry = telemetryEvidence(server);
if (telemetry) return { identity, observedAt: telemetry.receivedAt, healthy: true, actionable: false, reason: 'Container healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
const probed = snapshot ? snapshot.game : await rconForMonitoring(server, true).catch(error => {
@@ -280,16 +280,21 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
const oldEnough = Date.parse(observation.observedAt) - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
if (problem.state === 'open' && observation.actionable && oldEnough && problem.failures >= 3 && maintenanceAllowed('health')) {
// A missing container has nothing to restart. Recreate it through
// the normal rebuild queue, with its identity and safety checks.
const action = observation.containerState === 'missing' ? 'rebuild' : 'restart';
const result = db().transaction(() => {
const next = requestWatchdogLifecycle(server, 'restart', problem.id);
if (next.ok) addAction(problem, next.deploymentId, 'restart');
const next = requestWatchdogLifecycle(server, action, problem.id);
if (next.ok) addAction(problem, next.deploymentId, action);
return next;
})();
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error);
}
}
// Missing-container recovery belongs to the health policy. Do not turn
// absent files into drift or bypass the sustained health-failure window.
// Unknown container health is an observation failure, not proof that a rebuild is safe.
if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
if (observation.containerState === 'missing' || (!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
// File trees change far less often than liveness. Persist the last successful
// scan per registration/process so health checks do not rescan every minute.
// Manual checks and new lifecycle submissions always get fresh file evidence.
src/routes/maintenance/+page.svelte
+1 -1
View File
@@ -53,7 +53,7 @@
let checkedAt = $state<number | null>(null);
const tabs = [{ id: 'overview', label: 'Overview' }, { id: 'policies', label: 'Policies & schedules' }, { id: 'activity', label: 'Activity' }];
const policies: { id: MaintenancePolicy; title: string; description: string }[] = [
{ id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Rebuilding deletes instance files and installs the latest promoted setup.' },
{ id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Confirmed missing containers go directly to rebuild. Rebuilding deletes instance files and installs the latest promoted setup.' },
{ id: 'drift', title: 'Configuration drift', description: 'Rebuild servers whose managed files differ from the promoted setup. Record the differences in a linked Campaign. Rebuilding deletes instance files.' },
{ id: 'updates', title: 'Automatic Plutonium updates', description: 'Update outdated running servers on restart. One automatic attempt per published revision; failed verification requires inspection. Disabled by default.' }
];
src/routes/maintenance/[id]/+page.svelte
+3 -2
View File
@@ -44,7 +44,8 @@
if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.';
if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.';
if (incident.state === 'recovering') return 'Recovery is in progress. Follow the provisioning operations below for eligibility checks, progress, and health verification.';
return 'Maintenance detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server.';
if (incident.kind === 'health' && evidence.containerState?.toLowerCase() === 'missing') return 'Maintenance confirmed that the container is missing. When health recovery is enabled and Maintenance is not paused, three observations over at least two minutes trigger a direct rebuild without a player-count check. Capacity limits and protected work still apply.';
return 'Maintenance detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server. Offline hosts and unknown container states wait for another check.';
});
async function load() {
@@ -114,7 +115,7 @@
<h2>{incident.summary}</h2>
<p>{explanation}</p>
{#if activeOperation && (canRetry || canDismiss)}<p>Complete or cancel the active provisioning operation before retrying or dismissing this incident.</p>{/if}
{#if !incident.server_deleted_at}{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}{/if}
{#if !incident.server_deleted_at}{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery restarts an existing container, then rebuilds if needed. A confirmed missing container goes directly to rebuild. Rebuilding deletes any remaining instance files and installs the latest promoted setup.</p>{/if}{/if}
<dl>
<div><dt>First detected</dt><dd>{$timeFormat.dateTime(incident.first_seen_at)}</dd></div>
<div><dt>Last observed</dt><dd>{$timeFormat.dateTime(incident.last_seen_at)}</dd></div>