diff --git a/docs/guides/collections.md b/docs/guides/collections.md index b5c6d5f..9a5608a 100644 --- a/docs/guides/collections.md +++ b/docs/guides/collections.md @@ -304,6 +304,27 @@ before continuing eligible interrupted startups. Unknown command results retain their original command ID and occupied slots until the host confirms the outcome. Missing server registrations and later destruction attempts require inspection. +### Recovering a completed rebuild generation + +A completed rebuild records the replacement generation before EC verifies health. +If fresh host inventory disagrees, verification pauses with the controller and +host generations in the operation's recovery message. + +For an existing mismatch, open the **completed rebuild's Provisioning detail** +(even if that operation was later cancelled), then choose **Reconcile replacement +generation**. EC requires a completed command for the original host and immutable +instance ID, the command's requested replacement generation, and matching host +inventory observed within the last 30 seconds. Wait for that host to reconnect +and report fresh inventory if the action is unavailable. A server name alone is +not sufficient evidence. + +Reconciliation preserves cancellation and submitted commands. Use **Resume health +verification** when offered, or request a new lifecycle operation after resolving +any remaining active operation. Fresh health checks, log reads, and new lifecycle +commands use the replacement generation. EC requires fresh container and game +evidence before reporting the server online. The agent continues to reject stale +commands; an admin player-safety override does not bypass generation checks. + ## API Collections retain the existing `/api/blueprints` endpoints and `blueprintId` / @@ -351,6 +372,9 @@ there while preserving query parameters. - `GET /api/deployments/:id`: includes `recipe`, `blueprintName`, and `canResumeVerification` and `canRetryWithRebuild` for lifecycle operations. - `POST /api/deployments/:id/resume`: resume eligible creation health checks. +- `POST /api/provisioning/:id/reconcile-generation`: reconcile a completed rebuild + using fresh inventory from its original host. Returns 409 when identity or + completion evidence cannot be validated; repeated application is safe. - `POST /api/provisioning/:id/retry-rebuild`: request a new rebuild for an eligible failed creation and link it from the original operation. diff --git a/src/lib/provisioning-detail.ts b/src/lib/provisioning-detail.ts index f12bd4b..9ea0384 100644 --- a/src/lib/provisioning-detail.ts +++ b/src/lib/provisioning-detail.ts @@ -117,6 +117,13 @@ export interface ProvisioningCapacity { }[]; } export interface ProvisioningDetail { + generationRecovery?: { + expectedGeneration: number; + replacementGeneration: number; + observedGeneration: number | null; + available: boolean; + reason: string | null; + } | null; canOverrideSafety?: boolean; safetyOverrideUnavailableReason?: string | null; safetyOverride?: { actorId: string; actorName: string; confirmedAt: string } | null; diff --git a/src/lib/server/db/migrations.ts b/src/lib/server/db/migrations.ts index b019540..1d3cdc3 100644 --- a/src/lib/server/db/migrations.ts +++ b/src/lib/server/db/migrations.ts @@ -120,7 +120,21 @@ export const migrations: readonly Migration[] = [ { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank }, { version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials }, { version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions }, - { version: 23, name: 'host temperature history', run: migrateTemperatures } + { version: 23, name: 'host temperature history', run: migrateTemperatures }, + { + version: 24, + name: 'rebuild generation evidence boundary', + run: (handle) => { + if ( + !(handle.pragma('table_info(native_instances)') as { name: string }[]).some( + (column) => column.name === 'generation_reconciled_at' + ) + ) + handle.exec( + 'ALTER TABLE native_instances ADD COLUMN generation_reconciled_at TEXT' + ); + } + } ]; export const databaseVersion = migrations.length; diff --git a/src/lib/server/fleet/lifecycle-evidence.ts b/src/lib/server/fleet/lifecycle-evidence.ts index 9f323d2..2109426 100644 --- a/src/lib/server/fleet/lifecycle-evidence.ts +++ b/src/lib/server/fleet/lifecycle-evidence.ts @@ -7,9 +7,10 @@ export function latestGameSubmission(server: string): string | null { .prepare( `SELECT MAX(at) AS at FROM ( SELECT MAX(submitted_at) AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild') - UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)` + UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=? + UNION ALL SELECT generation_reconciled_at AS at FROM native_instances WHERE server=? AND retired_at IS NULL)` ) - .get(server, server) as { at: string | null } + .get(server, server, server) as { at: string | null } ).at; } @@ -21,7 +22,8 @@ export function latestGameSubmissions(): Map { SELECT s.name AS server, (SELECT MAX(submitted_at) FROM server_deployments WHERE server=s.name AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')) AS at FROM servers s - UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server` + UNION ALL SELECT server, requested_at AS at FROM restart_rollout + UNION ALL SELECT server, generation_reconciled_at AS at FROM native_instances WHERE retired_at IS NULL) GROUP BY server` ) .all() as { server: string; at: string | null }[]; return new Map(rows.map((row) => [row.server, row.at])); diff --git a/src/lib/server/provisioning/cancellation.ts b/src/lib/server/provisioning/cancellation.ts index b76fd29..dc0be06 100644 --- a/src/lib/server/provisioning/cancellation.ts +++ b/src/lib/server/provisioning/cancellation.ts @@ -128,6 +128,9 @@ export function unsettledCancelledCommand( WHERE d.server=? AND d.server_deleted_at IS NULL AND d.state='cancelled' AND (o.state NOT IN ('completed','failed') OR (o.state='completed' AND o.id=n.operation_id AND n.files_json IS NOT NULL AND n.applied_at IS NULL) + OR (o.state='completed' AND o.id=n.operation_id AND o.kind='instance.rebuild' + AND EXISTS (SELECT 1 FROM native_instances i WHERE i.id=n.instance_id + AND i.retired_at IS NULL AND i.generation<=o.generation)) OR (o.state='failed' AND o.id=n.operation_id AND o.kind='instance.rebuild' AND json_type(o.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL)) AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id AND c.cleanup_id=?) diff --git a/src/lib/server/provisioning/history.ts b/src/lib/server/provisioning/history.ts index b978db5..be5e141 100644 --- a/src/lib/server/provisioning/history.ts +++ b/src/lib/server/provisioning/history.ts @@ -14,6 +14,7 @@ import { fleetSettings } from '../settings.js'; import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js'; import { readProgress, recordProgress, type ProgressUpdate } from './progress.js'; import { bindNativeOperation } from './native.js'; +import { rebuildGenerationRecovery } from './rebuild-receipt.js'; import { workerActivity } from '../pipeline/activity.js'; import { canCancelCreation } from './cancel-creation.js'; import { @@ -337,6 +338,7 @@ export function getDeployment( canResumeCleanup: canResumeCleanup(id), canResumeRebuild: canResumeRebuild(id), canRetryWithRebuild: canRetryWithRebuild(id), + generationRecovery: rebuildGenerationRecovery(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null, progress: readProgress(row), diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 2669107..ffd8fc5 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -70,7 +70,11 @@ import { recordProgress, type CheckInput } from './progress.js'; import { trackActivity } from '../pipeline/activity.js'; import { fleetSettings } from '../settings.js'; import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js'; -import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js'; +import { + probeServerStatus, + probePlayerEvidence, + latestGameSubmission +} from '../fleet/status-evidence.js'; import { monitoringIdentity } from '../fleet/health-observations.js'; import { handoffVerifiedHealth } from './health-handoff.js'; import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js'; @@ -85,6 +89,12 @@ import { reconcileHostDrains } from './host-drain.js'; import { hostMigration, isRebalanceMigration, processMigrationRemoval } from './migrations.js'; import { safetyOverride, safetyOverrideCapability } from './safety-override.js'; import { runtimeLaunchSql } from '../fleet/runtime-config.js'; +import { + reconcileCompletedRebuild, + assertVerificationGeneration, + GenerationRecoveryRequired, + observedGeneration +} from './rebuild-receipt.js'; export type NativeSelection = import('../../hosts.js').NativePlacement; type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection }; @@ -813,27 +823,11 @@ export function reconcileNativeProvisioningReceipt(id: string): void { finishCreationCancellation(job.id); return; } - if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) { - db() - .prepare('UPDATE native_instances SET generation=generation+1 WHERE id=?') - .run(instance.id); - // Replacements queued while the delivered rebuild was settling bound the - // previous generation. Only untouched, newer jobs may follow this receipt. - if (job.state === 'cancelled') - db() - .prepare( - `UPDATE native_provisioning SET generation=? - WHERE instance_id=? AND generation=? AND operation_id IS NULL AND stage_operation_id IS NULL - AND deployment_id IN (SELECT newer.id FROM server_deployments newer - WHERE newer.rowid>(SELECT rowid FROM server_deployments WHERE id=?) - AND newer.server_deleted_at IS NULL AND newer.state IN ('queued','waiting'))` - ) - .run(job.generation + 1, instance.id, job.generation, id); - } else if ( - instance.generation !== job.generation && - !(operation.kind === 'instance.rebuild' && instance.generation === job.generation + 1) - ) - return; + if (operation.kind === 'instance.rebuild') { + // An older applied receipt must never undo a later successful replacement. + if (job.applied_at && instance.generation > operation.generation! + 1) return; + reconcileCompletedRebuild(id); + } else if (instance.generation !== job.generation) return; applyReceipt(job, (operation.result ?? {}) as Record); db() .prepare( @@ -1032,10 +1026,12 @@ export async function processNativeProvisioning( `SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id LEFT JOIN ec_host_operations o ON o.id=n.operation_id + LEFT JOIN native_instances i ON i.id=n.instance_id AND i.retired_at IS NULL LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id LEFT JOIN ec_host_operations removal ON removal.id=m.remove_operation_id WHERE d.state='cancelled' AND d.server_deleted_at IS NULL AND ( - (o.state='completed' AND (n.applied_at IS NULL OR d.action='destroy')) + (o.state='completed' AND (n.applied_at IS NULL OR d.action='destroy' + OR (o.kind='instance.rebuild' AND i.generation=o.generation))) OR (o.state='failed' AND o.kind='instance.rebuild' AND json_type(o.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL) OR (o.state='failed' AND EXISTS (SELECT 1 FROM staged_file_changes f JOIN provisioning_sources s ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server @@ -1061,7 +1057,8 @@ export async function processNativeProvisioning( `SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.applied_at,n.recovery_step,q.selection FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id LEFT JOIN native_creation_intents q ON q.deployment_id=d.id - WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid` + WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') + AND COALESCE(CASE WHEN json_valid(d.observation_json) THEN json_extract(d.observation_json,'$.blocker.code') END,'')!='generation-recovery' ORDER BY d.started_at,d.rowid` ) .all() as Job[]; // One host's observation must not prevent another job being admitted. @@ -1075,7 +1072,27 @@ export async function processNativeProvisioning( () => processJob(job, eligibility) ) .catch((error) => { - if (stillActive(job)) hold(job, (error as Error).message); + if (!stillActive(job)) return; + if ( + error instanceof GenerationRecoveryRequired || + /generation has been superseded or removed/i.test((error as Error).message) + ) { + const instance = nativeInstance(job.server); + const message = + error instanceof GenerationRecoveryRequired + ? error.message + : `Generation recovery required: controller expects ${instance?.generation ?? job.generation}, host reports ${instance ? (observedGeneration(instance) ?? 'unknown') : 'unknown'}. Reconcile the completed rebuild before resuming verification.`; + db() + .prepare( + "UPDATE server_deployments SET state='interrupted',error=? WHERE id=?" + ) + .run(message, job.id); + deploymentStage(job.id, message, { + blocker: { code: 'generation-recovery', message, automatic: false }, + checks: [], + recovery: { state: 'held', operationId: job.id, message } + }); + } else hold(job, (error as Error).message); }) .finally(() => { activeJobs.delete(job.id); @@ -1328,6 +1345,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { job.instance_id = instance.id; job.generation = instance.generation; } + // Reconcile before comparing the job snapshot or issuing any observations. + // Historical jobs can have a stale binding even though their command is correct. + if (job.operation_id) { + const receipt = getHostOperationStatus(job.operation_id); + if (receipt?.kind === 'instance.rebuild' && receipt.state === 'completed') { + instance = reconcileCompletedRebuild(job.id); + job.generation = receipt.generation!; + } + } const migration = hostMigration(job.id); const rebuilding = job.recovery_step === 'rebuild' || @@ -1336,7 +1362,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { !instance || instance.id !== job.instance_id || (instance.generation !== job.generation && - !(rebuilding && instance.generation === job.generation + 1)) + !(rebuilding && job.operation_id && instance.generation === job.generation + 1)) ) { finishDeployment(job.id, { ok: false, @@ -1493,12 +1519,6 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { })(); return; } - if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) { - db() - .prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?') - .run(job.generation + 1, instance.id, job.generation); - instance = nativeInstance(job.server)!; - } applyReceipt(job, (operation.result ?? {}) as Record); db() .prepare( @@ -1542,6 +1562,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { blocker: null, checks }); + assertVerificationGeneration(instance, operation.updatedAt); const verificationIdentity = monitoringIdentity(job.server); const health = await nativeRequest<{ healthy: boolean; state: string }>( instance, @@ -1601,7 +1622,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { return; } deploymentStage(job.id, 'Checking game status', { phase: 'verify-game', blocker: null }); - const status = await probeServerStatus(job.server, operation.updatedAt); + const submission = latestGameSubmission(job.server); + const status = await probeServerStatus( + job.server, + submission && submission > operation.updatedAt ? submission : operation.updatedAt + ); if (!stillActive(job)) return; check( { @@ -2255,8 +2280,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { ) : queueHostOperation(instance!.host_id, kind, payload, operationOptions); db() - .prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?') - .run(operation.id, job.id); + .prepare( + 'UPDATE native_provisioning SET operation_id=?,generation=? WHERE deployment_id=?' + ) + .run(operation.id, instance!.generation, job.id); for (const file of files) db() .prepare( diff --git a/src/lib/server/provisioning/rebuild-receipt.ts b/src/lib/server/provisioning/rebuild-receipt.ts new file mode 100644 index 0000000..3a1b331 --- /dev/null +++ b/src/lib/server/provisioning/rebuild-receipt.ts @@ -0,0 +1,173 @@ +import { db } from '../db/index.js'; +import { nativeInstance, type NativeInstance } from '../backends/native.js'; +import { getHost, getHostOperationStatus } from '../hosts/index.js'; +import { recordProgress } from './progress.js'; + +export class GenerationRecoveryRequired extends Error {} + +/** Inventory is evidence only when it belongs to this host and immutable instance. */ +export function observedGeneration(instance: NativeInstance, notBefore?: string): number | null { + const host = getHost(instance.host_id); + if (!host?.connected) return null; + const states = host.inventory.instanceStates?.filter((state) => { + const age = Date.now() - Date.parse(state.observedAt); + return ( + state.instanceId === instance.id && + age >= 0 && + age <= 30_000 && + (!notBefore || Date.parse(state.observedAt) >= Date.parse(notBefore)) + ); + }); + return states?.length === 1 ? states[0].generation : null; +} + +function receipt(id: string) { + const binding = db() + .prepare( + `SELECT d.server,n.instance_id,n.generation,n.operation_id + FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id + WHERE d.id=? AND d.server_deleted_at IS NULL + AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id)` + ) + .get(id) as + | { server: string; instance_id: string; generation: number; operation_id: string } + | undefined; + const instance = binding && nativeInstance(binding.server); + const operation = binding?.operation_id && getHostOperationStatus(binding.operation_id); + if ( + !binding || + !instance || + !operation || + operation.kind !== 'instance.rebuild' || + operation.state !== 'completed' || + operation.error || + instance.id !== binding.instance_id || + operation.instanceId !== instance.id || + operation.hostId !== instance.host_id || + !Number.isSafeInteger(operation.generation) || + operation.generation! < 1 + ) { + throw new GenerationRecoveryRequired( + 'Recovery requires a completed rebuild for this host and immutable instance ID. Inspect the original host command.' + ); + } + const requested = db() + .prepare( + `SELECT COALESCE( + json_extract(COALESCE(p.payload,o.payload),'$.nextGeneration'), + json_extract(COALESCE(p.payload,o.payload),'$.newGeneration')) AS generation + FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?` + ) + .get(operation.id) as { generation: number | null }; + const generation = requested.generation; + const result = operation.result as { instanceId?: unknown; generation?: unknown } | null; + if ( + !Number.isSafeInteger(generation) || + generation !== operation.generation! + 1 || + (result?.instanceId !== undefined && result.instanceId !== instance.id) || + (result?.generation !== undefined && result.generation !== generation) || + ![operation.generation, generation].includes(instance.generation) + ) { + throw new GenerationRecoveryRequired( + 'The completed rebuild does not match the registered instance and requested replacement generation. Inspect its host command; no generation was adopted.' + ); + } + return { binding, instance, operation, generation: generation! }; +} + +/** Read-only availability for the operator recovery action, including old cancelled jobs. */ +export function rebuildGenerationRecovery(id: string) { + try { + const { binding, instance, operation, generation } = receipt(id); + const observed = observedGeneration(instance, operation.updatedAt); + if (instance.generation === generation && binding.generation === operation.generation) + return null; + return { + expectedGeneration: instance.generation, + replacementGeneration: generation, + observedGeneration: observed, + available: observed === generation, + reason: + observed === generation + ? null + : `Waiting for fresh inventory from the original host confirming generation ${generation} for this instance.` + }; + } catch { + return null; + } +} + +/** The durable command, not a potentially stale job snapshot, owns the transition. */ +export function reconcileCompletedRebuild(id: string, requireInventory = false): NativeInstance { + return db().transaction(() => { + const { binding, instance, operation, generation } = receipt(id); + const observed = observedGeneration(instance, operation.updatedAt); + if ( + (requireInventory && observed !== generation) || + (observed !== null && observed !== generation) + ) { + throw new GenerationRecoveryRequired( + `Generation recovery required: controller expects ${instance.generation}, completed rebuild requests ${generation}, host reports ${observed ?? 'unknown'}. Refresh the host inventory and reconcile the completed rebuild.` + ); + } + const changed = instance.generation !== generation; + const repaired = binding.generation !== operation.generation; + if (changed) { + const at = new Date().toISOString(); + db() + .prepare( + `UPDATE native_instances SET generation=?,generation_reconciled_at=? + WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL` + ) + .run(generation, at, instance.id, instance.host_id, instance.generation); + db() + .prepare( + `UPDATE servers SET container_state='unknown',container_checked_at=?, + last_checked_at=NULL,last_players=NULL,last_roster=NULL WHERE name=?` + ) + .run(at, instance.server); + } + db() + .prepare( + 'UPDATE native_provisioning SET generation=? WHERE deployment_id=? AND operation_id=?' + ) + .run(operation.generation, id, operation.id); + // Untouched successors may follow a cancelled, delivered rebuild. Already + // submitted commands retain their immutable target and must never be retargeted. + db() + .prepare( + `UPDATE native_provisioning SET generation=? WHERE instance_id=? AND generation=? + AND operation_id IS NULL AND stage_operation_id IS NULL + AND deployment_id IN (SELECT newer.id FROM server_deployments newer + WHERE newer.rowid>(SELECT rowid FROM server_deployments WHERE id=?) + AND newer.submitted_at IS NULL AND newer.server_deleted_at IS NULL + AND newer.state IN ('queued','waiting'))` + ) + .run(generation, instance.id, instance.generation, id); + if (changed || repaired) + recordProgress(id, { + checks: [], + event: { + kind: 'generation-reconciled', + message: `Recorded replacement generation ${generation}; fresh health verification is required.`, + details: { + operationId: operation.id, + hostId: instance.host_id, + instanceId: instance.id, + previousGeneration: instance.generation, + generation + } + } + }); + return nativeInstance(instance.server)!; + })(); +} + +export function assertVerificationGeneration(instance: NativeInstance, notBefore: string): void { + const observed = observedGeneration(instance, notBefore); + if (observed !== null && observed !== instance.generation) { + throw new GenerationRecoveryRequired( + `Generation recovery required: controller expects ${instance.generation}, host reports ${observed}. Reconcile the completed rebuild before resuming verification.` + ); + } +} diff --git a/src/lib/server/retention.ts b/src/lib/server/retention.ts index ace9ab1..3ceb9a7 100644 --- a/src/lib/server/retention.ts +++ b/src/lib/server/retention.ts @@ -31,7 +31,7 @@ const unsettledDeployment = `d.state NOT IN ${terminal} OR (a.id=n.operation_id AND a.state='completed' AND ( (n.applied_at IS NULL AND COALESCE(json_array_length(n.files_json),0)>0) OR (d.action='destroy' AND i.retired_at IS NULL) - OR (a.kind='instance.rebuild' AND i.retired_at IS NULL AND i.generation=n.generation))))) + OR (a.kind='instance.rebuild' AND i.retired_at IS NULL AND i.generation<=a.generation))))) OR EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id WHERE m.deployment_id=d.id AND (a.state NOT IN ('completed','failed') OR (a.state='completed' AND m.removed_at IS NULL)))`; // Recovery replaces operation_id with its latest command. The original launch diff --git a/src/lib/ui/provisioning-refresh.ts b/src/lib/ui/provisioning-refresh.ts index 5a97c47..2cb30b3 100644 --- a/src/lib/ui/provisioning-refresh.ts +++ b/src/lib/ui/provisioning-refresh.ts @@ -1,5 +1,5 @@ export type ProvisioningMutation = - 'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override'; + 'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override' | 'reconcile-generation'; export interface ProvisioningRefreshState { id: string; diff --git a/src/routes/api/provisioning/[id]/reconcile-generation/+server.ts b/src/routes/api/provisioning/[id]/reconcile-generation/+server.ts new file mode 100644 index 0000000..75d6b04 --- /dev/null +++ b/src/routes/api/provisioning/[id]/reconcile-generation/+server.ts @@ -0,0 +1,26 @@ +import { json } from '@sveltejs/kit'; +import { reconcileCompletedRebuild } from '$lib/server/provisioning/rebuild-receipt.js'; +import { reconcileNativeProvisioningReceipt } from '$lib/server/provisioning/native.js'; +import { db } from '$lib/server/db/index.js'; +import { recordProgress } from '$lib/server/provisioning/progress.js'; + +export function POST({ params }: { params: { id: string } }) { + try { + const instance = db().transaction(() => { + const instance = reconcileCompletedRebuild(params.id, true); + reconcileNativeProvisioningReceipt(params.id); + recordProgress(params.id, { + blocker: null, + checks: [], + event: { + kind: 'generation-recovery', + message: `Replacement generation ${instance.generation} reconciled with fresh host inventory. Resume health verification or request a new lifecycle operation.` + } + }); + return instance; + })(); + return json({ ok: true, generation: instance.generation }); + } catch (error) { + return json({ error: (error as Error).message }, { status: 409 }); + } +} diff --git a/src/routes/provisioning/[id]/+page.svelte b/src/routes/provisioning/[id]/+page.svelte index 03482cf..a49c5d3 100644 --- a/src/routes/provisioning/[id]/+page.svelte +++ b/src/routes/provisioning/[id]/+page.svelte @@ -324,6 +324,27 @@ )}.

{/if} + {#if detail.generationRecovery}
+

Replacement generation recovery

+

+ Controller generation {detail.generationRecovery.expectedGeneration}; + completed rebuild generation {detail.generationRecovery + .replacementGeneration}; host generation {detail.generationRecovery + .observedGeneration ?? 'unknown'}. +

+ {#if detail.generationRecovery.reason}

+ {detail.generationRecovery.reason} +

{/if} + {#if writable}{/if} +
{/if} {#if writable && (detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canRetryWithRebuild || detail.canCancel || detail.canCancelCreation || canOverride)}
{#if canOverride}
diff --git a/tests/unit/auth-access.test.ts b/tests/unit/auth-access.test.ts index 59c0d3b..5efbf02 100644 --- a/tests/unit/auth-access.test.ts +++ b/tests/unit/auth-access.test.ts @@ -90,6 +90,7 @@ describe('management route permissions', () => { ['/api/rebalancing', 'POST'], ['/api/provisioning/[id]/cancel', 'POST'], ['/api/provisioning/[id]/resume', 'POST'], + ['/api/provisioning/[id]/reconcile-generation', 'POST'], ['/api/deployment-collections', 'POST'], ['/api/deployment-collections/preview', 'POST'], ['/api/deployment-collections/[id]/cancel', 'POST'], diff --git a/tests/unit/database-migrations.test.ts b/tests/unit/database-migrations.test.ts index b16657c..94ebef9 100644 --- a/tests/unit/database-migrations.test.ts +++ b/tests/unit/database-migrations.test.ts @@ -291,3 +291,18 @@ it('adds provisioning report invalidation at version 14 without invalidating for upgraded.exec('DELETE FROM server_deployments'); expect(revision()).toBeGreaterThan(completed); }); + +it('adds the generation evidence boundary to a version 23 database without changing instance identity', async () => { + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const handle = db(); + handle.exec( + "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s','host',2,'lib','image',1,'2026-10-14T00:00:00Z')" + ); + handle.exec('ALTER TABLE native_instances DROP COLUMN generation_reconciled_at'); + handle.pragma('user_version = 23'); + closeDb(); + expect( + db().prepare('SELECT id,generation,generation_reconciled_at FROM native_instances').get() + ).toEqual({ id: 'instance', generation: 2, generation_reconciled_at: null }); + expect(db().pragma('foreign_key_check')).toEqual([]); +}); diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index 1876ecb..695aaf9 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -465,7 +465,16 @@ describe('provisioning health handoff', () => { .run(old, old, old, job.name); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000); - const observedAt = new Date().toISOString(); + if (action === 'rebuild') { + // A new generation now establishes its own durable evidence boundary. + // Observe game health strictly after that transition. + const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); + vi.mocked(nativeRequest).mockImplementationOnce(async () => { + vi.setSystemTime(Date.now() + 1); + return { healthy: true, state: 'running', exists: true } as never; + }); + } + const observedAt = new Date(Date.now() + (action === 'rebuild' ? 1 : 0)).toISOString(); const roster = [ { num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 } ]; @@ -4288,3 +4297,276 @@ describe('game key pool launch lifecycle', () => { expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 }); }); }); + +describe('completed rebuild generation receipts (#155)', () => { + async function rebuild() { + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + const requested = requestLifecycle(job.name, 'rebuild'); + if (!requested.ok) throw new Error(requested.error); + await advance(); + return { + job, + id: requested.deploymentId, + operation: (await operationFor(requested.deploymentId))! + }; + } + + async function inventoryGeneration( + instanceId: string, + generation: number, + observedAt = new Date().toISOString(), + targetHost = hostId + ) { + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + heartbeat(targetHost, { + ...inventory(), + instanceStates: [ + { instanceId, generation, state: 'running', healthy: true, observedAt } + ] + }); + } + + it('repairs an older job binding from the actual command before verification, including duplicate receipts and restart', async () => { + const { job, id } = await rebuild(); + await acknowledge(id); + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + const next = requestLifecycle(job.name, 'rebuild'); + if (!next.ok) throw new Error(next.error); + await advance(); + const operation = (await operationFor(next.deploymentId))!; + expect(operation.generation).toBe(2); + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + // Historical job snapshots could stay on generation 1 while the submitted + // command correctly targeted 2 -> 3. The old completion path kept 2. + db() + .prepare('UPDATE native_provisioning SET generation=1 WHERE deployment_id=?') + .run(next.deploymentId); + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, operation.id, { + state: 'completed', + result: { instanceId: operation.instanceId, generation: 3, healthy: true } + }); + await inventoryGeneration(operation.instanceId!, 3); + closeDb(); + const { nativeRequest, nativeInstance } = + await import('../../src/lib/server/backends/native.js'); + vi.mocked(nativeRequest).mockClear(); + remote.state = 'starting'; + await advance(); + expect(nativeInstance(job.name)?.generation).toBe(3); + expect(nativeRequest).toHaveBeenCalledWith( + expect.objectContaining({ id: operation.instanceId, generation: 3 }), + 'instance.inspect' + ); + expect( + db() + .prepare( + 'SELECT generation,applied_at FROM native_provisioning WHERE deployment_id=?' + ) + .get(next.deploymentId) + ).toEqual({ generation: 2, applied_at: expect.any(String) }); + const { reconcileNativeProvisioningReceipt } = + await import('../../src/lib/server/provisioning/native.js'); + const before = db() + .prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?') + .get(operation.instanceId); + reconcileNativeProvisioningReceipt(next.deploymentId); + closeDb(); + reconcileNativeProvisioningReceipt(next.deploymentId); + expect(nativeInstance(job.name)?.generation).toBe(3); + expect( + db() + .prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?') + .get(operation.instanceId) + ).toEqual(before); + expect((await detail(next.deploymentId)).state).toBe('awaiting-health'); + for (const observation of await observations(job.name)) + expect(observation.health).not.toBe('online'); + const { latestGameSubmission, latestGameSubmissions } = + await import('../../src/lib/server/fleet/lifecycle-evidence.js'); + expect(latestGameSubmission(job.name)).toBe( + (before as { generation_reconciled_at: string }).generation_reconciled_at + ); + expect(latestGameSubmissions().get(job.name)).toBe(latestGameSubmission(job.name)); + remote.state = 'running'; + await advance(); + expect((await detail(next.deploymentId)).state).toBe('done'); + const restart = requestLifecycle(job.name, 'restart'); + if (!restart.ok) throw new Error(restart.error); + await advance(); + expect(await operationFor(restart.deploymentId)).toMatchObject({ generation: 3 }); + }); + + it.each(['before', 'after'] as const)( + 'reconciles delayed completion reported %s cancellation across restart', + async (when) => { + const { job, id, operation } = await rebuild(); + const { reportHostOperation, pendingHostOperations } = + await import('../../src/lib/server/hosts/index.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); + const { closeDb } = await import('../../src/lib/server/db/index.js'); + pendingHostOperations(hostId); + if (when === 'before') + reportHostOperation(hostId, operation.id, { state: 'completed' }); + expect(cancelProvisioningJob(id)).toBe(true); + closeDb(); + if (when === 'after') reportHostOperation(hostId, operation.id, { state: 'completed' }); + await advance(); + await advance(); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)?.generation).toBe(2); + expect((await detail(id)).state).toBe('cancelled'); + } + ); + + it('recovers an already-applied cancelled mismatch through the API and targets replacement logs, health and removal', async () => { + const { job, id, operation } = await rebuild(); + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, operation.id, { + state: 'completed', + result: { instanceId: operation.instanceId, generation: 2 } + }); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); + expect(cancelProvisioningJob(id)).toBe(true); + const { db } = await import('../../src/lib/server/db/index.js'); + db() + .prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?') + .run(operation.updatedAt, id); + await inventoryGeneration(operation.instanceId!, 2); + const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + expect(getDeployment(id)?.generationRecovery).toMatchObject({ + expectedGeneration: 1, + observedGeneration: 2, + available: true + }); + const { POST } = + await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js'); + expect(await POST({ params: { id } }).json()).toEqual({ ok: true, generation: 2 }); + expect(POST({ params: { id } }).status).toBe(200); + expect((await detail(id)).state).toBe('cancelled'); + expect(getDeployment(id)?.generationRecovery).toBeNull(); + const backend = await import('../../src/lib/server/backends/native.js'); + vi.mocked(backend.nativeRequest).mockRestore(); + const hosts = await import('../../src/lib/server/hosts/index.js'); + vi.spyOn(hosts, 'waitHostOperationStatus').mockImplementation(async (id) => + hosts.reportHostOperationStatus(hostId, id, { + state: 'completed', + result: { healthy: true, state: 'running' } + }) + ); + await backend.nativeRequest(backend.nativeInstance(job.name)!, 'logs.read', { + file: 'console.log' + }); + expect(await backend.nativeHealth(job.name)).toMatchObject({ ok: true }); + expect( + db() + .prepare( + "SELECT generation FROM ec_host_operations WHERE kind IN ('logs.read','instance.inspect')" + ) + .all() + .every((row) => (row as { generation: number }).generation === 2) + ).toBe(true); + const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); + const removal = destroyServer(job.name); + if (!removal.ok || !removal.deploymentId) throw new Error('Removal not queued'); + await advance(); + expect(await operationFor(removal.deploymentId)).toMatchObject({ + generation: 2, + kind: 'instance.remove' + }); + }); + + it.each([ + 'host', + 'instance', + 'result-instance', + 'result-generation', + 'payload', + 'stale-inventory', + 'missing-inventory', + 'inventory-instance' + ] as const)('rejects recovery evidence with %s mismatch', async (mismatch) => { + const { job, id, operation } = await rebuild(); + const { reportHostOperation, createHost } = + await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, operation.id, { + state: 'completed', + result: { + instanceId: + mismatch === 'result-instance' ? 'another-instance' : operation.instanceId, + generation: mismatch === 'result-generation' ? 3 : 2 + } + }); + const { db } = await import('../../src/lib/server/db/index.js'); + if (mismatch === 'host') + db() + .prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?') + .run(createHost({ name: 'Other host' }).id, operation.id); + if (mismatch === 'instance') + db() + .prepare('UPDATE ec_host_operations SET instance_id=? WHERE id=?') + .run('another-instance', operation.id); + if (mismatch === 'payload') + db() + .prepare( + "UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?" + ) + .run(operation.id); + if (mismatch !== 'missing-inventory') + await inventoryGeneration( + mismatch === 'inventory-instance' ? 'another-instance' : operation.instanceId!, + 2, + mismatch === 'stale-inventory' ? '2000-01-01T00:00:00Z' : new Date().toISOString() + ); + const { POST } = + await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js'); + expect(POST({ params: { id } }).status).toBe(409); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)?.generation).toBe(1); + }); + + it('pauses after an agent stale-generation rejection even when inventory is unavailable', async () => { + const { id, operation } = await rebuild(); + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, operation.id, { state: 'completed' }); + const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); + vi.mocked(nativeRequest).mockClear(); + vi.mocked(nativeRequest).mockRejectedValueOnce( + new Error('Instance generation has been superseded or removed') + ); + await advance(); + expect((await detail(id)).state).toBe('interrupted'); + expect((await detail(id)).error).toContain('controller expects 2, host reports unknown'); + await advance(); + expect(nativeRequest).toHaveBeenCalledTimes(1); + }); + + it('stops verification retries on a confirmed generation mismatch and keeps expected/observed evidence', async () => { + const { id, operation } = await rebuild(); + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, operation.id, { state: 'completed' }); + await inventoryGeneration(operation.instanceId!, 3); + const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); + vi.mocked(nativeRequest).mockClear(); + await advance(); + const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const before = getDeployment(id)!; + expect(before.deployment.state).toBe('interrupted'); + expect(before.progress?.blocker).toMatchObject({ + code: 'generation-recovery', + automatic: false, + message: expect.stringContaining( + 'controller expects 1, completed rebuild requests 2, host reports 3' + ) + }); + await advance(); + await advance(); + expect(nativeRequest).not.toHaveBeenCalled(); + expect(getDeployment(id)?.events).toEqual(before.events); + }); +}); diff --git a/tests/unit/provisioning-history-performance.test.ts b/tests/unit/provisioning-history-performance.test.ts index 9827b20..6da088b 100644 --- a/tests/unit/provisioning-history-performance.test.ts +++ b/tests/unit/provisioning-history-performance.test.ts @@ -157,7 +157,7 @@ describe('provisioning reads with retained lifecycle history', () => { }; expectIndexed( () => expect(latestGameSubmission('current')).toBe('2026-01-01'), - ['current', 'current'] + ['current', 'current', 'current'] ); expectIndexed(() => expect(hasActiveProvisioning('current')).toBe(true), ['current', '']); expectIndexed(() => expect(housePartyTargets()).toEqual(['current'])); diff --git a/tests/unit/retention.test.ts b/tests/unit/retention.test.ts index c4d3590..cab842b 100644 --- a/tests/unit/retention.test.ts +++ b/tests/unit/retention.test.ts @@ -710,3 +710,35 @@ it.each(['cancelled', 'failed'])( ).toEqual({ history_expired_at: expect.any(String) }); } ); + +it('retains completed rebuild evidence when a stale job binding hid an unreconciled generation', async () => { + const a = await api(); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const host = hosts.createHost({ name: 'Host' }); + a.db + .prepare( + "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,2,'lib','image',1,?)" + ) + .run(host.id, old); + a.db + .prepare( + "INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')" + ) + .run(old, old); + const command = hosts.queueHostOperation( + host.id, + 'instance.rebuild', + { nextGeneration: 3 }, + { instanceId: 'instance', generation: 2 } + ); + hosts.reportHostOperation(host.id, command.id, { state: 'completed' }); + a.db + .prepare( + "INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,applied_at) VALUES('cancelled','instance',1,?,?)" + ) + .run(command.id, old); + a.saveFleetSettings({ ...a.fleetSettings(), provisioningKeepDays: 1, historyKeepCount: 0 }); + expect(a.pruneHistory('provisioning')).toBe(0); + a.db.prepare("UPDATE native_instances SET generation=3 WHERE id='instance'").run(); + expect(a.pruneHistory('provisioning')).toBe(1); +});