Rebuild missing containers during maintenance recovery

This commit is contained in:
2026-10-08 13:33:01 -04:00
parent 368e457dc6
commit b3fa486886
9 changed files with 201 additions and 25 deletions
+1 -1
View File
@@ -53,7 +53,7 @@
let checkedAt = $state<number | null>(null);
const tabs = [{ id: 'overview', label: 'Overview' }, { id: 'policies', label: 'Policies & schedules' }, { id: 'activity', label: 'Activity' }];
const policies: { id: MaintenancePolicy; title: string; description: string }[] = [
{ id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Rebuilding deletes instance files and installs the latest promoted setup.' },
{ id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Confirmed missing containers go directly to rebuild. Rebuilding deletes instance files and installs the latest promoted setup.' },
{ id: 'drift', title: 'Configuration drift', description: 'Rebuild servers whose managed files differ from the promoted setup. Record the differences in a linked Campaign. Rebuilding deletes instance files.' },
{ id: 'updates', title: 'Automatic Plutonium updates', description: 'Update outdated running servers on restart. One automatic attempt per published revision; failed verification requires inspection. Disabled by default.' }
];
+3 -2
View File
@@ -44,7 +44,8 @@
if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.';
if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.';
if (incident.state === 'recovering') return 'Recovery is in progress. Follow the provisioning operations below for eligibility checks, progress, and health verification.';
return 'Maintenance detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server.';
if (incident.kind === 'health' && evidence.containerState?.toLowerCase() === 'missing') return 'Maintenance confirmed that the container is missing. When health recovery is enabled and Maintenance is not paused, three observations over at least two minutes trigger a direct rebuild without a player-count check. Capacity limits and protected work still apply.';
return 'Maintenance detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server. Offline hosts and unknown container states wait for another check.';
});
async function load() {
@@ -114,7 +115,7 @@
<h2>{incident.summary}</h2>
<p>{explanation}</p>
{#if activeOperation && (canRetry || canDismiss)}<p>Complete or cancel the active provisioning operation before retrying or dismissing this incident.</p>{/if}
{#if !incident.server_deleted_at}{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}{/if}
{#if !incident.server_deleted_at}{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery restarts an existing container, then rebuilds if needed. A confirmed missing container goes directly to rebuild. Rebuilding deletes any remaining instance files and installs the latest promoted setup.</p>{/if}{/if}
<dl>
<div><dt>First detected</dt><dd>{$timeFormat.dateTime(incident.first_seen_at)}</dd></div>
<div><dt>Last observed</dt><dd>{$timeFormat.dateTime(incident.last_seen_at)}</dd></div>