Recover unhealthy containers during create and rebuild

src/lib/server/backends/schema.ts

@@ -46,6 +46,9 @@ export function migrateNativeInstances(handle: Database.Database): void {
     if (!(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some(column => column.name === 'runtime_password')) {
         handle.exec('ALTER TABLE ec_host_migrations ADD COLUMN runtime_password TEXT');
     }
+    if (!(handle.pragma('table_info(native_provisioning)') as { name: string }[]).some(column => column.name === 'recovery_step')) {
+        handle.exec("ALTER TABLE native_provisioning ADD COLUMN recovery_step TEXT CHECK (recovery_step IN ('restart','rebuild'))");
+    }
     if (!handle.prepare("SELECT 1 FROM settings WHERE key='native_file_plans_v1'").get()) {
         handle.transaction(() => {
             handle.exec(`UPDATE native_provisioning SET files_json=(SELECT json_group_array(json_remove(value,'$.contentBase64'))

src/lib/server/hosts/index.ts

@@ -466,10 +466,13 @@ export function listHostOperationHistory(page = 1, filter: HostOperationFilter =
 
 /** Bound each cleanup pass so retained history cannot stall monitoring or receipts. */
 export function pruneHostObservations(): number { return pruneHistory('observation'); }
-/** Heartbeats can tell an executing create to stop at its next safe command boundary. */
+/** Heartbeats stop either launch generation at its next safe command boundary. */
 export function pendingCreationCancellations(hostId: string): string[] {
-    return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
-        WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`).all(hostId) as { id: string }[]).map(row => row.id);
+    const rows = database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS creation_id,
+        json_extract(payload,'$.recoveryRebuildOperationId') AS recovery_id FROM ec_host_operations
+        WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`)
+        .all(hostId) as { creation_id: string; recovery_id: string | null }[];
+    return rows.flatMap(row => row.recovery_id ? [row.creation_id, row.recovery_id] : [row.creation_id]);
 }
 type QueueHostOperationOptions = { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string };
 export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: QueueHostOperationOptions = {}): HostOperation {
@@ -537,8 +540,11 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
     // busy indefinitely. Drain the old batch by withholding ALL further work;
     // heartbeats and receipts still flow, and the new capability releases this hold.
     // This must live in the controller so the fix also works before an agent update.
-    if (!host.inventory.capabilities?.includes('instance.cancel-create') && database().prepare(`SELECT 1 FROM ec_host_operations
-        WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
+    if (database().prepare(`SELECT 1 FROM ec_host_operations
+        WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')
+        AND (NOT ? OR (json_extract(payload,'$.recoveryRebuildOperationId') IS NOT NULL AND NOT ?)) LIMIT 1`)
+        .get(hostId, Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false),
+            Number(host.inventory.capabilities?.includes('instance.unhealthy-recovery') ?? false))) return [];
     // Accepted/running/unknown commands keep the same immutable ID. The agent
     // consults its durable journal before acknowledging or executing a replay.
     const healthOnly = `(kind IN ('instance.inspect','rcon.status-batch') OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
@@ -557,6 +563,9 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
             AND (kind!='instance.cancel-create' OR ?)
             AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
                 AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
+            AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cancellation
+                JOIN native_provisioning creation ON creation.deployment_id=cancellation.creation_id
+                WHERE creation.operation_id=o.id)
             ${lane === 'health' ? "AND (state NOT IN ('accepted','running') OR MAX(updated_at,COALESCE(last_delivered_at,'')) <= ?)" : ''}
             ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 WHEN kind='plutonium.prepare' THEN 2 ELSE 1 END,
                 COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)

src/lib/server/provisioning/cancel-creation.ts

@@ -2,14 +2,16 @@ import { pauseScalingForOperation } from '../autoscaling/ownership.js';
 import { db } from '../db/index.js';
 import { releaseServerKey } from '../fleet/server-keys.js';
 import { nativeInstance } from '../backends/native.js';
-import { queueHostOperation, releaseHostPorts } from '../hosts/index.js';
+import { getHostOperationStatus, queueHostOperation, releaseHostPorts } from '../hosts/index.js';
 import { beginDeployment, cancelRemoval, deploymentStage, finishDeployment } from './history.js';
 import { recordProgress } from './progress.js';
 
 export function canCancelCreation(id: string): boolean {
     return !!db().prepare(`SELECT 1 FROM server_deployments d
         LEFT JOIN native_provisioning n ON n.deployment_id=d.id
-        LEFT JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
+        LEFT JOIN native_instances i ON i.id=n.instance_id AND i.retired_at IS NULL
+            AND (i.generation=n.generation OR (n.recovery_step='rebuild' AND i.generation=n.generation+1
+                AND n.applied_at IS NOT NULL))
         WHERE d.id=? AND d.action='create' AND d.server_deleted_at IS NULL
         AND (i.id IS NOT NULL OR (n.deployment_id IS NULL
             AND EXISTS (SELECT 1 FROM native_creation_intents intent WHERE intent.deployment_id=d.id)
@@ -29,9 +31,9 @@ export function cancelCreation(id: string, triggeredBy = 'ui', pausePolicy = tru
         if (prior) return prior.cleanup_id;
         if (!canCancelCreation(id)) return null;
         if (pausePolicy) pauseScalingForOperation(id);
-        const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d
+        const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id,n.generation,n.recovery_step FROM server_deployments d
             LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
-            { server: string; operation_id: string | null; stage_operation_id: string | null };
+            { server: string; operation_id: string | null; stage_operation_id: string | null; generation: number | null; recovery_step: 'restart' | 'rebuild' | null };
         const instance = nativeInstance(job.server);
         // Replace a removal still waiting on occupancy, including one whose asynchronous probe is outstanding.
         for (const removal of db().prepare("SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL").all(job.server) as { id: string }[]) {
@@ -54,8 +56,30 @@ export function cancelCreation(id: string, triggeredBy = 'ui', pausePolicy = tru
             finishCreationCancellation(cleanupId);
         } else {
             if (!instance) throw new Error('The original native instance is unavailable.');
-            const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', { createOperationId: job.operation_id ?? id },
-                { instanceId: instance.id, generation: instance.generation, idempotencyKey: cleanupId });
+            let generation = instance.generation;
+            const payload: { createOperationId: string; recoveryRebuildOperationId?: string; recoveryGeneration?: number } = { createOperationId: job.operation_id ?? id };
+            if (job.recovery_step) {
+                // Recovery replaces operation_id, but cancellation must retain the
+                // immutable ownership label from the original launch.
+                const original = db().prepare(`SELECT id FROM ec_host_operations WHERE host_id=? AND instance_id=?
+                    AND generation=? AND kind='instance.create' ORDER BY rowid DESC LIMIT 1`)
+                    .get(instance.host_id, instance.id, job.generation) as { id: string } | undefined;
+                if (!original || !job.generation) throw new Error('The original creation command is unavailable.');
+                payload.createOperationId = original.id;
+                generation = job.generation;
+                if (job.recovery_step === 'rebuild') {
+                    const recovery = job.operation_id ? getHostOperationStatus(job.operation_id) : null;
+                    if (!recovery || recovery.kind !== 'instance.rebuild' || recovery.instanceId !== instance.id || recovery.generation !== generation) {
+                        throw new Error('The creation recovery command is unavailable.');
+                    }
+                    // A delivered rebuild may have created the next generation
+                    // before its receipt arrives. Fence and remove both identities.
+                    payload.recoveryRebuildOperationId = recovery.id;
+                    payload.recoveryGeneration = generation + 1;
+                }
+            }
+            const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', payload,
+                { instanceId: instance.id, generation, idempotencyKey: cleanupId });
             db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, cleanupId);
             db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), cleanupId);
             deploymentStage(cleanupId, 'Waiting for host agent to cancel creation and remove its partial instance', { phase: 'container-action', blocker: null,

src/lib/server/provisioning/concurrency.ts

@@ -14,7 +14,7 @@ export function lifecycleHolders(): Holder[] {
     // Find capacity owners through their active state or outstanding command.
     // A correlated OR over d used to walk every retained job and fetch its agent
     // receipt on every admission, even when all that history was completed.
-    return db().prepare(`SELECT d.id,d.action,i.host_id,m.target_host_id,
+    return db().prepare(`SELECT d.id,CASE WHEN n.recovery_step='rebuild' THEN 'rebuild' ELSE d.action END AS action,i.host_id,m.target_host_id,
         CASE WHEN m.remove_operation_id IS NOT NULL AND m.removed_at IS NULL THEN 1 ELSE 0 END AS removing
         FROM server_deployments d
         LEFT JOIN native_provisioning n ON n.deployment_id=d.id

src/lib/server/provisioning/history.ts

@@ -107,7 +107,9 @@ function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): P
         pools.push({ kind: 'startup', limit: settings.createConcurrency, used: holders.length, holders,
             requestOrder: order < 0 ? null : order + 1, ownsSlot: holders.some(holder => holder.id === deployment.id) });
     }
-    if (['start', 'restart', 'rebuild'].includes(deployment.action)) {
+    const creationRecovery = deployment.action === 'create' && db().prepare(`SELECT 1 FROM native_provisioning
+        WHERE deployment_id=? AND recovery_step IS NOT NULL`).get(deployment.id);
+    if (['start', 'restart', 'rebuild'].includes(deployment.action) || creationRecovery) {
         const holders = db().prepare(`SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL
             ORDER BY requested_at, server`).all() as { id: string | null; server: string }[];
         pools.push({ kind: 'restart', limit: settings.restartConcurrency, used: holders.length, holders,

src/lib/server/provisioning/launch-preparation.ts

@@ -15,6 +15,7 @@ export interface NativeLaunchInput {
     rconPassword: string;
     rebuild: boolean;
     safetyOverride?: boolean;
+    unhealthyRecovery?: boolean;
     /** Rebalance replays recorded deployed bytes instead of current promoted files. */
     frozenRecipeId?: string;
 }

src/lib/server/provisioning/launch-worker.ts

@@ -44,7 +44,9 @@ parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeL
                 PLUTO_SERVER_KEY: input.serverKey },
             files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path),
                 contentSha256: file.sha256, sha256: file.sha256 })),
-            ...(input.rebuild ? { nextGeneration: instance.generation + 1, ...(input.safetyOverride ? { safetyOverride: true } : {}) } : {})
+            ...(input.rebuild ? { nextGeneration: instance.generation + 1,
+                ...(input.safetyOverride ? { safetyOverride: true } : {}),
+                ...(input.unhealthyRecovery ? { unhealthyRecovery: true } : {}) } : {})
         };
         const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value));
         const blobs = [...new Map(recipe.files.map(file => [file.sha256,

src/lib/server/provisioning/native.ts

@@ -2,7 +2,7 @@ import { assertScalingOwnership } from '../autoscaling/ownership.js';
 import { removalAuthorization } from '../autoscaling/authorization.js';
 import { scalingIntent } from '../autoscaling/admission.js';
 import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js';
-import { concurrencyBlocker, preparingJobs, committedRebalanceStartup } from './concurrency.js';
+import { concurrencyBlocker, preparingJobs, committedRebalanceStartup, lifecycleHolders } from './concurrency.js';
 import { isIP } from 'node:net';
 import { setImmediate as yieldToController } from 'node:timers/promises';
 import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
@@ -39,12 +39,15 @@ import { cancelCreation, finishCreationCancellation } from './cancel-creation.js
 import { reconcileHostDrains } from './host-drain.js';
 import { hostMigration, isRebalanceMigration, processMigrationRemoval } from './migrations.js';
 import { safetyOverride, safetyOverrideCapability } from './safety-override.js';
+import { runtimeLaunchSql } from '../fleet/runtime-config.js';
 
 export type NativeSelection = import('../../hosts.js').NativePlacement;
 type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
 interface Job { selection?: string | null; id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
-    instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; applied_at: string | null }
+    instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; applied_at: string | null;
+    recovery_step: 'restart' | 'rebuild' | null }
 interface FilePlan { id: number; path: string; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
+const unhealthyRecoveryCapability = 'instance.unhealthy-recovery';
 
 /** Validate a launch request without selecting a host or consuming capacity. */
 export function nativeLaunchSelection(input: Record<string, unknown>, game: string): NativeSelection {
@@ -242,7 +245,7 @@ function storedFiles(job: Job): FilePlan[] {
 
 /** Agent receipts are the authoritative before images; do not read a controller mount. */
 function applyReceipt(job: Job, result: Record<string, unknown>) {
-    if (job.applied_at) return;
+    if (job.applied_at || job.recovery_step) return;
     const plans = storedFiles(job);
     if (!plans.length) return;
     const before = result.before as { id?: number; path: string; contentBase64: string | null; contentSha256?: string }[] | undefined;
@@ -276,7 +279,7 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
 
 /** Retain confirmed file snapshots and instance identity even after observation is cancelled. */
 export function reconcileNativeProvisioningReceipt(id: string): void {
-    const job = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.operation_id,n.applied_at
+    const job = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.operation_id,n.applied_at,n.recovery_step
         FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
         WHERE d.id=? AND d.server_deleted_at IS NULL`).get(id) as Job | undefined;
     if (!job) return;
@@ -309,6 +312,13 @@ export function reconcileNativeProvisioningReceipt(id: string): void {
     if (operation?.state !== 'completed') return;
     const instance = nativeInstance(job.server);
     if (!instance || instance.id !== job.instance_id) return;
+    if (job.state === 'cancelled' && job.action === 'create'
+        && db().prepare('SELECT 1 FROM native_creation_cancellations WHERE creation_id=?').get(id)) {
+        // Cleanup owns both generations of a cancelled recovery rebuild. Do not
+        // advance the registry underneath its already-bound removal command.
+        db().prepare('UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?').run(operation.updatedAt, id);
+        return;
+    }
     if (job.action === 'destroy') {
         if (instance.generation !== job.generation) return;
         db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(operation.updatedAt, instance.id);
@@ -318,7 +328,7 @@ export function reconcileNativeProvisioningReceipt(id: string): void {
         finishCreationCancellation(job.id);
         return;
     }
-    if (job.action === 'rebuild' && !hostMigration(id) && instance.generation === job.generation) {
+    if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) {
         db().prepare('UPDATE native_instances SET generation=generation+1 WHERE id=?').run(instance.id);
         // Replacements queued while the delivered rebuild was settling bound the
         // previous generation. Only untouched, newer jobs may follow this receipt.
@@ -328,7 +338,7 @@ export function reconcileNativeProvisioningReceipt(id: string): void {
                 WHERE newer.rowid>(SELECT rowid FROM server_deployments WHERE id=?)
                 AND newer.server_deleted_at IS NULL AND newer.state IN ('queued','waiting'))`)
             .run(job.generation + 1, instance.id, job.generation, id);
-    } else if (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1)) return;
+    } else if (instance.generation !== job.generation && !(operation.kind === 'instance.rebuild' && instance.generation === job.generation + 1)) return;
     applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
     db().prepare('UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?').run(operation.updatedAt, id);
 }
@@ -342,10 +352,78 @@ function complete(job: Job) {
     db().transaction(() => {
         db().prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
         db().prepare('UPDATE restart_rollout SET healthy_at=?,error=NULL WHERE operation_id=?').run(new Date().toISOString(), job.id);
+        if (job.recovery_step) recordProgress(job.id, { recovery: { state: 'released', operationId: job.id,
+            message: 'Container and game health verified after provisioning recovery.' } });
         finishDeployment(job.id, { ok: true });
     })();
 }
 
+/** Continue the same launch with at most one restart and one replacement. */
+function queueUnhealthyRecovery(job: Job, instance: NativeInstance): void {
+    db().transaction(() => {
+        if (!stillActive(job)) return;
+        const live = nativeInstance(job.server);
+        if (live?.id !== instance.id || live.generation !== instance.generation) return;
+        if (job.recovery_step === 'rebuild') {
+            const message = 'Container is still unhealthy after recovery restart and rebuild. Inspect the server before retrying.';
+            db().prepare('UPDATE restart_rollout SET error=? WHERE operation_id=?').run(message, job.id);
+            finishDeployment(job.id, { ok: false, message });
+            return;
+        }
+        const host = getHost(instance.host_id);
+        if (!host?.connected || host.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; }
+        if (!host.inventory.capabilities?.includes(unhealthyRecoveryCapability)) {
+            hold(job, 'Update this host’s agent to recover unhealthy containers during provisioning.'); return;
+        }
+        const step = job.recovery_step === 'restart' ? 'rebuild' : 'restart';
+        const settings = fleetSettings();
+        const slots = db().prepare(`SELECT i.host_id FROM restart_rollout r LEFT JOIN native_instances i
+            ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND (r.operation_id IS NULL OR r.operation_id!=?)`)
+            .all(job.id) as { host_id: string | null }[];
+        if (settings.restartConcurrency > 0 && slots.length >= settings.restartConcurrency) {
+            hold(job, 'Waiting for restart recovery capacity.'); return;
+        }
+        if (settings.restartConcurrencyPerHost > 0 && slots.filter(slot => slot.host_id === instance.host_id).length >= settings.restartConcurrencyPerHost) {
+            hold(job, 'Waiting for restart recovery capacity on this host.'); return;
+        }
+        let payload: Record<string, unknown> = { unhealthyRecovery: true };
+        if (step === 'rebuild') {
+            const beta = activeBetaRemovalBlocker(job.server);
+            if (beta) { hold(job, beta); return; }
+            if (job.action !== 'rebuild' && settings.rebuildConcurrency > 0) {
+                if (lifecycleHolders().filter(holder => holder.id !== job.id && holder.action === 'rebuild').length >= settings.rebuildConcurrency) {
+                    hold(job, 'Waiting for rebuild capacity.'); return;
+                }
+            }
+            assertReadyLibrary(instance.host_id, instance.library_id);
+            // Recover the files and credentials selected for this launch, even
+            // if new versions have been promoted while readiness was pending.
+            const launch = db().prepare(`SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o
+                JOIN native_instances i ON ${runtimeLaunchSql}
+                LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id
+                WHERE i.id=? AND o.state='completed' AND o.history_expired_at IS NULL ORDER BY o.rowid DESC LIMIT 1`)
+                .get(instance.id) as { payload: string } | undefined;
+            if (!launch) { hold(job, 'Waiting for the original launch payload before recovery rebuild.'); return; }
+            payload = { ...JSON.parse(launch.payload), nextGeneration: instance.generation + 1, unhealthyRecovery: true };
+            delete payload.safetyOverride;
+        }
+        const operation = queueHostOperationSerialized(instance.host_id, `instance.${step}`, Buffer.from(JSON.stringify(payload)),
+            { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:unhealthy:${step}` },
+            { name: instance.server, unhealthyRecovery: true });
+        const at = new Date().toISOString();
+        db().prepare('UPDATE native_provisioning SET operation_id=?,generation=?,recovery_step=?,applied_at=NULL WHERE deployment_id=?')
+            .run(operation.id, instance.generation, step, job.id);
+        db().prepare("UPDATE server_deployments SET state='running',submitted_at=?,error=NULL,finished_at=NULL WHERE id=?").run(at, job.id);
+        db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
+            DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, at, job.id);
+        const message = `Container unhealthy; ${step === 'restart' ? 'forcing a restart' : 'forcing a rebuild after restart recovery failed'}.`;
+        deploymentStage(job.id, message, { phase: 'container-action', blocker: null, checks: [],
+            command: { state: 'attempted', message: `Recovery ${step} queued as agent operation ${operation.id}.` },
+            recovery: { state: 'held', operationId: job.id, message: 'Recovery runs within this provisioning operation.' },
+            event: { kind: 'unhealthy-recovery', message, details: { step, operationId: operation.id, generation: instance.generation } } });
+    })();
+}
+
 export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null,
     options: { wait?: boolean } = {}): Promise<void> {
     ensureHostSchema(db());
@@ -369,7 +447,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
         }
     }
     reconcileHostDrains();
-    const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.applied_at,q.selection
+    const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.applied_at,n.recovery_step,q.selection
         FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id
         LEFT JOIN native_creation_intents q ON q.deployment_id=d.id
         WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
@@ -485,7 +563,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         job.instance_id = instance.id; job.generation = instance.generation;
     }
     const migration = hostMigration(job.id);
-    if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && !migration && instance.generation === job.generation + 1))) {
+    const rebuilding = job.recovery_step === 'rebuild' || (!job.recovery_step && job.action === 'rebuild' && !migration);
+    if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(rebuilding && instance.generation === job.generation + 1))) {
         finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
     }
     if (migration && !migration.removed_at) {
@@ -499,7 +578,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     if (job.operation_id) {
         const operation = getHostOperationStatus(job.operation_id);
-        if (operation?.kind === 'instance.cancel-create' && !['completed', 'failed'].includes(operation.state) && !host.inventory.capabilities?.includes('instance.cancel-create')) {
+        const recoveryCleanup = operation?.kind === 'instance.cancel-create' && !!db().prepare(`SELECT 1 FROM ec_host_operations
+            WHERE id=? AND json_extract(payload,'$.recoveryRebuildOperationId') IS NOT NULL`).get(operation.id);
+        if (operation?.kind === 'instance.cancel-create' && !['completed', 'failed'].includes(operation.state)
+            && (!host.inventory.capabilities?.includes('instance.cancel-create') || (recoveryCleanup && !host.inventory.capabilities?.includes(unhealthyRecoveryCapability)))) {
             hold(job, hostAgentUpdatesEnabled(host.inventory)
                 ? 'Waiting for the host agent update. New command delivery is paused so its current batch can finish and the automatic updater can run. Cleanup resumes after the updated agent connects.'
                 : 'Update the host agent to support creation cancellation: rerun the current installer on this host. Automatic updates are not enabled. New command delivery is paused; cleanup and port release await the updated agent.'); return;
@@ -514,7 +596,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 })();
                 return;
             }
-            if (migration) {
+            if (migration && !job.recovery_step) {
                 if (Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
                     db().transaction(() => {
                         db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id);
@@ -552,15 +634,17 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             })();
             return;
         }
-        if (job.action === 'rebuild' && !migration && instance.generation === job.generation) {
+        if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) {
             db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation);
             instance = nativeInstance(job.server)!;
         }
         applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
+        db().prepare('UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?').run(operation.updatedAt, job.id);
         if (job.state !== 'awaiting-health') {
             db().prepare("UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?").run(job.id);
-            recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` },
-                checkpoint: { key: 'container-action', label: `Agent confirmed ${job.action}`, at: operation.updatedAt } });
+            const action = job.recovery_step ? `recovery ${job.recovery_step}` : job.action;
+            recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${action}; verifying the resulting instance.` },
+                checkpoint: { key: job.recovery_step ? `recovery-${job.recovery_step}` : 'container-action', label: `Agent confirmed ${action}`, at: operation.updatedAt } });
         }
         const checks: CheckInput[] = [];
         const check = (value: CheckInput, phase: import('../../provisioning-detail.js').ProvisioningPhase) => {
@@ -580,7 +664,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             complete(job); return;
         }
         check({ key: 'container', label: 'Container readiness verified', source: 'agent', result: health.healthy ? 'pass' : 'waiting', actual: health.state }, 'verify-container');
-        if (!health.healthy) { hold(job, `Waiting for container readiness: ${health.state}`); return; }
+        if (!health.healthy) {
+            if (health.state === 'unhealthy' && (job.action === 'create' || job.action === 'rebuild')) queueUnhealthyRecovery(job, instance);
+            else hold(job, `Waiting for container readiness: ${health.state}`);
+            return;
+        }
         deploymentStage(job.id, 'Checking game status', { phase: 'verify-game', blocker: null });
         const status = await probeServerStatus(job.server, operation.updatedAt);
         if (!stillActive(job)) return;
@@ -669,6 +757,12 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
         const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
         if (!stillActive(job)) return false;
+        if (job.action === 'rebuild' && health.state === 'unhealthy') {
+            if (!getHost(instance.host_id)?.inventory.capabilities?.includes(unhealthyRecoveryCapability)) {
+                hold(job, 'Update this host’s agent to recover unhealthy containers during provisioning.'); return false;
+            }
+            return true;
+        }
         if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
             const occupancy = await probePlayerEvidence(job.server);
             if (!stillActive(job)) return false;
@@ -708,7 +802,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             preparedPassword = config.rcon.password || randomBytes(32).toString('hex');
             prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
                 instance, serverKey: preparedKey, rconPassword: preparedPassword,
-                rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety, frozenRecipeId });
+                rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety,
+                unhealthyRecovery: host.inventory.capabilities?.includes(unhealthyRecoveryCapability), frozenRecipeId });
             if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
         }
         const commit = db().transaction(() => {

src/lib/server/retention.ts

@@ -22,12 +22,26 @@ const unsettledDeployment = `d.state NOT IN ${terminal}
         (a.state NOT IN ('completed','failed') OR (a.id=n.operation_id AND a.state='completed' AND (
             (n.applied_at IS NULL AND COALESCE(json_array_length(n.files_json),0)>0)
             OR (d.action='destroy' AND i.retired_at IS NULL)
-            OR (d.action='rebuild' AND i.retired_at IS NULL AND i.generation=n.generation
-                AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id))))))
+            OR (a.kind='instance.rebuild' AND i.retired_at IS NULL AND i.generation=n.generation)))))
     OR EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id
         WHERE m.deployment_id=d.id AND (a.state NOT IN ('completed','failed') OR (a.state='completed' AND m.removed_at IS NULL)))`;
+// Recovery replaces operation_id with its latest command. The original launch
+// still supplies rebuild files and the ownership label used by creation cleanup.
+const recoveryLaunch = `EXISTS (SELECT 1 FROM native_provisioning n
+    JOIN server_deployments d ON d.id=n.deployment_id
+    JOIN native_instances i ON i.id=n.instance_id AND i.retired_at IS NULL
+    WHERE n.recovery_step IS NOT NULL AND d.server_deleted_at IS NULL AND o.instance_id=i.id AND o.host_id=i.host_id
+    AND ((o.kind='instance.create' AND o.generation=n.generation)
+        OR (o.kind='instance.rebuild' AND o.generation+1=n.generation))
+    AND ((${unsettledDeployment}) OR (d.action='create' AND d.state IN ('failed','cancelled')
+        AND i.generation BETWEEN n.generation AND n.generation+1
+        AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server
+            AND newer.server_deleted_at IS NULL AND newer.rowid>d.rowid AND newer.state!='cancelled'
+            AND NOT (newer.action='destroy' AND newer.state='waiting'
+                AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=newer.id AND p.operation_id IS NOT NULL))))))`;
 const unreferencedOperation = `NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE o.id IN (n.operation_id,n.stage_operation_id))
     AND NOT EXISTS (SELECT 1 FROM native_instances i WHERE i.retired_at IS NULL AND ${runtimeLaunchSql})
+    AND NOT (${recoveryLaunch})
     AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.remove_operation_id=o.id)
     AND NOT EXISTS (SELECT 1 FROM ec_host_decommissions d WHERE d.operation_id=o.id)
     AND o.kind NOT IN ('runtime.pull','library.download','library.import','library.remove','host.decommission','host.rotate-credential')
@@ -97,6 +111,7 @@ function policies(): Policy[] {
             failed: "o.state='failed'",
             eligible: `o.state IN ('completed','failed') AND o.kind IN ('instance.create','instance.rebuild','instance.restart','instance.start','files.stage','files.read')
                 AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='operation' AND c.record_id=o.id AND c.protected=1)
+                AND NOT (${recoveryLaunch})
                 AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
                     WHERE o.id IN (n.operation_id,n.stage_operation_id) AND ((${unsettledDeployment}) OR EXISTS (
                         SELECT 1 FROM retention_controls c WHERE c.category IN ('provisioning','workflow-payload') AND c.record_id=d.id AND c.protected=1)))