diff --git a/docs/guides/collections.md b/docs/guides/collections.md index be0d197..a1e5a7e 100644 --- a/docs/guides/collections.md +++ b/docs/guides/collections.md @@ -287,12 +287,22 @@ server launch until the health checks pass. For failed or interrupted creations, **Resume health verification** appears only when the original server is still registered on its assigned port and no later -lifecycle operation has targeted that ID. It resumes observation only; it never -recreates or restarts a container. Missing servers and later destruction attempts -require inspection rather than replaying provisioning automatically. -After an application restart, the worker automatically resumes observation of -eligible interrupted startups so their occupied slots are retained. It never -repeats container creation or deployment as part of this recovery. +lifecycle operation has targeted that ID. Native commands must have completed +before readiness verification can resume. If the container is unhealthy, the +operation may continue its remaining automatic restart and rebuild recovery steps. + +When the host conclusively reports that the initial creation command failed, +**Retry with rebuild** starts a new provisioning operation for the same lobby and +port. It deletes the container and instance files and uses the latest promoted +setup. Normal player safety, capacity, and protected work checks still apply. +The original failure stays in history and links to the recovery operation. +This retry is unavailable for unknown command results, replacement instances, +cancelled creation cleanup, or a later operation on the same server. + +After an application restart, the worker reconciles the original host command +before continuing eligible interrupted startups. Unknown command results retain +their original command ID and occupied slots until the host confirms the outcome. +Missing server registrations and later destruction attempts require inspection. ## API @@ -339,7 +349,9 @@ there while preserving query parameters. - `POST /api/servers`: existing request plus optional `previewId`; the game and map must match that preview. `expectedName` remains supported. - `GET /api/deployments/:id`: includes `recipe`, `blueprintName`, and - `canResumeVerification` for lifecycle operations. + `canResumeVerification` and `canRetryWithRebuild` for lifecycle operations. - `POST /api/deployments/:id/resume`: resume eligible creation health checks. +- `POST /api/provisioning/:id/retry-rebuild`: request a new rebuild for an eligible + failed creation and link it from the original operation. Collection launches begin processing immediately after all requests are queued. diff --git a/src/lib/provisioning-detail.ts b/src/lib/provisioning-detail.ts index 32c58e8..c848330 100644 --- a/src/lib/provisioning-detail.ts +++ b/src/lib/provisioning-detail.ts @@ -101,6 +101,7 @@ export interface ProvisioningDetail { safetyOverride?: { actorId: string; actorName: string; confirmedAt: string } | null; canCancel?: boolean; canCancelCreation?: boolean; + canRetryWithRebuild?: boolean; cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null; backend?: 'native'; workerActivity?: WorkerActivity[]; diff --git a/src/lib/server/provisioning/history.ts b/src/lib/server/provisioning/history.ts index 9d713a7..4513716 100644 --- a/src/lib/server/provisioning/history.ts +++ b/src/lib/server/provisioning/history.ts @@ -14,6 +14,7 @@ import { canCancelCreation } from './cancel-creation.js'; import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js'; import { committedRebalanceStartup } from './concurrency.js'; import { nativeDeployment } from '../backends/native.js'; +import { ensureHostSchema } from '../hosts/schema.js'; import { safetyOverride, safetyOverrideAvailability } from './safety-override.js'; import { serverTitle } from '../../ui/server-labels.js'; import { provisioningQueueStatus } from '../../ui/provisioning-queue.js'; @@ -149,7 +150,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' } activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources, recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null, canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), - canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null, + canResumeRebuild: canResumeRebuild(id), canRetryWithRebuild: canRetryWithRebuild(id), + canCancelRemoval: canCancelRemoval(id), collection: collection ?? null, progress: readProgress(row), capacity: capacityFor(deployment, generatedAt), warningMessages: warningMessages(deployment.warnings), generatedAt }; })(); @@ -197,17 +199,43 @@ export function resumeCleanup(id: string): boolean { } export function canResumeVerification(id: string): boolean { + ensureHostSchema(db()); return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_creation_jobs j ON j.deployment_id = d.id - JOIN servers s ON s.name = j.server AND s.game_port = j.port - WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND (EXISTS ( - SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL) OR EXISTS ( - SELECT 1 FROM ec_host_migrations m JOIN native_provisioning n ON n.deployment_id=m.deployment_id - WHERE m.deployment_id=d.id AND m.removed_at IS NOT NULL AND n.operation_id IS NOT NULL)))) AND d.state IN ('failed', 'interrupted') + JOIN servers s ON s.name = j.server AND s.game_port = j.port AND s.destroyed_at IS NULL + LEFT JOIN native_provisioning n ON n.deployment_id=d.id + LEFT JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL + LEFT JOIN ec_host_operations o ON o.id=n.operation_id + WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('create','rebuild') + AND d.state IN ('failed', 'interrupted') + AND ((n.deployment_id IS NOT NULL AND i.port=j.port AND o.state='completed' + AND o.instance_id=i.id AND o.host_id=i.host_id AND o.generation=n.generation + AND ((o.kind IN ('instance.create','instance.restart') AND i.generation=n.generation) + OR (o.kind='instance.rebuild' AND i.generation IN (n.generation,n.generation+1)))) + OR (n.deployment_id IS NULL AND (d.action='create' OR EXISTS ( + SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id=d.id AND r.removed_at IS NOT NULL)))) AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`) .get(id); } +/** A conclusively failed initial create can enter the normal rebuild flow. */ +export function canRetryWithRebuild(id: string): boolean { + ensureHostSchema(db()); + return !!db().prepare(`SELECT d.id FROM server_deployments d + JOIN server_creation_jobs j ON j.deployment_id=d.id + JOIN servers s ON s.name=d.server AND s.game_port=j.port AND s.destroyed_at IS NULL + JOIN native_provisioning n ON n.deployment_id=d.id AND n.recovery_step IS NULL + JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL + AND i.generation=n.generation AND i.port=j.port + JOIN ec_host_operations o ON o.id=n.operation_id AND o.instance_id=i.id + AND o.host_id=i.host_id AND o.generation=n.generation + WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action='create' + AND d.state IN ('failed','interrupted') AND o.kind='instance.create' AND o.state='failed' + AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id) + AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)`) + .get(id); +} + export function canResumeRebuild(id: string): boolean { return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL @@ -226,7 +254,7 @@ export function resumeRebuild(id: string): boolean { })(); } -/** Resume observation only. Never repeat a container creation or restart. */ +/** Resume readiness after a completed command; unhealthy recovery still applies. */ export function resumeVerification(id: string): boolean { return db().transaction(() => { if (!canResumeVerification(id)) return false; diff --git a/src/lib/server/provisioning/index.ts b/src/lib/server/provisioning/index.ts index 05478c5..1b11247 100644 --- a/src/lib/server/provisioning/index.ts +++ b/src/lib/server/provisioning/index.ts @@ -1,6 +1,6 @@ // Request/observation boundary used by APIs, campaigns, beta tests, and schedules. // Execution is driven exclusively by the provisioning worker. -export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js'; +export { requestLifecycle, requestRetryWithRebuild, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js'; export type { LifecycleOperation, RestartSource } from './requests.js'; export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextLobbyNumber } from './instances.js'; export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js'; diff --git a/src/lib/server/provisioning/requests.ts b/src/lib/server/provisioning/requests.ts index 081ef93..460b796 100644 --- a/src/lib/server/provisioning/requests.ts +++ b/src/lib/server/provisioning/requests.ts @@ -3,7 +3,7 @@ import { currentRequestUsername } from '../auth/request-context.js'; import { maintenanceAllowed } from '../maintenance/policies.js'; import { db } from '../db/index.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; -import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js'; +import { beginDeployment, canRetryWithRebuild, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js'; import { queueRebuildServer } from './instances.js'; import { runtimeActions, type RuntimeAction } from '../../provisioning.js'; import { recordProgress } from './progress.js'; @@ -166,6 +166,23 @@ export function requestLifecycle(server: string, action: RuntimeAction) { type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string }; +/** Keep the failed launch immutable and let normal rebuild admission own recovery. */ +export function requestRetryWithRebuild(id: string): LifecycleRequestResult { + return db().transaction(() => { + if (!canRetryWithRebuild(id)) return { ok: false as const, status: 409, + error: 'This failed creation can no longer be retried with a rebuild. Refresh and inspect its current operation.' }; + const original = provisioningOperation(id)!; + const result = requestLifecycle(original.server, 'rebuild'); + if (!result.ok) return result; + recordProgress(id, { recovery: { state: 'transferred', operationId: result.deploymentId, + message: 'A new rebuild was requested for this server. The failed creation and its error are preserved.' }, + event: { kind: 'retry-requested', message: 'Retry with rebuild requested', details: { operationId: result.deploymentId } } }); + recordProgress(result.deploymentId, { event: { kind: 'retry-requested', + message: 'Rebuild requested after failed creation', details: { previousOperationId: id } } }); + return result; + })(); +} + function watchdogRequestGuard(server: string, incidentId: string): Extract | null { const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined; if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' }; diff --git a/src/lib/ui/provisioning-presentation.ts b/src/lib/ui/provisioning-presentation.ts index 7663a9e..a3ef7fb 100644 --- a/src/lib/ui/provisioning-presentation.ts +++ b/src/lib/ui/provisioning-presentation.ts @@ -76,11 +76,13 @@ export function provisioningOutcome(detail: ProvisioningDetail) { let description: string; if (detail.canResumeCleanup) description = 'The remote deployment was deleted. Resume the remaining local cleanup without repeating container removal.'; else if (detail.canResumeRebuild) description = 'The remote deployment was deleted. Resume local cleanup, then create a fresh instance in the reserved server slot.'; - else if (detail.canResumeVerification) description = 'Resume observation of the existing instance. This does not create, restart, or rebuild a container.'; + else if (detail.canResumeVerification) description = 'Resume readiness checks for the existing instance. An unhealthy container may continue through this operation’s remaining restart and rebuild recovery steps.'; + else if (detail.canRetryWithRebuild) description = 'The host reported that creation failed. Retry with rebuild starts a new attempt for the same lobby and port using the latest promoted setup.'; else if (detail.cancellation) description = detail.cancellation.state === 'done' ? 'Creation was cancelled. Its instance files, container, and port reservations have been removed.' : 'Creation was cancelled. Cleanup is tracked separately below; ports remain reserved until the host confirms removal.'; else if (operation.state === 'cancelled') description = 'This request is cancelled. Recorded prior work is preserved below; remaining work will not run.'; + else if (failure && progress?.recovery?.state === 'transferred' && progress.recovery.operationId && progress.recovery.operationId !== operation.id) description = 'A recovery operation has been requested for this server. Follow the linked operation below; this record keeps the original failure.'; else if (failure) description = progress?.command?.state === 'uncertain' ? 'The remote response was uncertain. Inspect the recorded result and server before requesting another operation.' : 'Review the recorded failure and execution evidence before submitting another operation.'; @@ -162,6 +164,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi if (!record(value) || !record(value.deployment)) return false; if (value.canCancel !== undefined && typeof value.canCancel !== 'boolean') return false; if (value.canCancelCreation !== undefined && typeof value.canCancelCreation !== 'boolean') return false; + if (value.canRetryWithRebuild !== undefined && typeof value.canRetryWithRebuild !== 'boolean') return false; if (value.cancellation !== undefined && value.cancellation !== null && !(record(value.cancellation) && text(value.cancellation.cleanupId) && text(value.cancellation.state) && text(value.cancellation.stage) && nullableText(value.cancellation.error))) return false; diff --git a/src/lib/ui/provisioning-refresh.ts b/src/lib/ui/provisioning-refresh.ts index 05f08bb..18d3543 100644 --- a/src/lib/ui/provisioning-refresh.ts +++ b/src/lib/ui/provisioning-refresh.ts @@ -1,4 +1,4 @@ -export type ProvisioningMutation = 'resume' | 'cancel' | 'cancel-creation' | 'override'; +export type ProvisioningMutation = 'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override'; export interface ProvisioningRefreshState { id: string; diff --git a/src/routes/api/provisioning/[id]/retry-rebuild/+server.ts b/src/routes/api/provisioning/[id]/retry-rebuild/+server.ts new file mode 100644 index 0000000..6975c71 --- /dev/null +++ b/src/routes/api/provisioning/[id]/retry-rebuild/+server.ts @@ -0,0 +1,10 @@ +import { json } from '@sveltejs/kit'; +import { requestRetryWithRebuild } from '$lib/server/provisioning/index.js'; +import { kickProvisioning } from '$lib/server/pipeline/worker.js'; + +export function POST({ params }: { params: { id: string } }) { + const result = requestRetryWithRebuild(params.id); + if (!result.ok) return json({ error: result.error }, { status: result.status }); + void kickProvisioning().catch(err => console.error('[provisioning] rebuild retry kick failed:', err)); + return json(result, { status: 202 }); +} diff --git a/src/routes/provisioning/[id]/+page.svelte b/src/routes/provisioning/[id]/+page.svelte index 08d9234..8dd2d6c 100644 --- a/src/routes/provisioning/[id]/+page.svelte +++ b/src/routes/provisioning/[id]/+page.svelte @@ -59,6 +59,17 @@ }); onMount(() => pollVisible(() => { now = Date.now(); return controller.refresh(); }, 5000, false)); onDestroy(() => controller.dispose()); + async function retryWithRebuild() { + if (!deployment || !detail?.canRetryWithRebuild || view.busy) return; + const id = deployment.id; + const confirmed = await confirmAction({ + title: 'Retry creation with a rebuild?', + message: `Rebuild ${serverTitle(deployment.server)} (${deployment.server}) using the latest promoted setup. This keeps the same lobby and port, deletes its container and all instance files, and starts a new provisioning operation.\n\nPlayer safety, capacity limits, and protected work checks still apply. The original failure remains in this operation’s history.`, + confirmLabel: 'Retry with rebuild', danger: true + }); + if (confirmed && view.id === id && detail?.canRetryWithRebuild) await controller.act('retry-rebuild', + 'Rebuild retry requested. Follow the linked recovery operation below for progress.'); + } async function overrideSafety() { if (!deployment || !canOverride || view.busy || detail?.safetyOverrideUnavailableReason) return; const id = deployment.id; @@ -107,10 +118,11 @@ {#if detail.warningMessages.length}

Recorded warnings

    {#each detail.warningMessages as warning}
  • {warning}
    • {/each}
{/if} {#if detail.safetyOverride}

Admin override enabled

Player safety checks bypassed for this operation. Confirmed by {detail.safetyOverride.actorName} · .

{/if} - {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancel || detail.canCancelCreation || canOverride} + {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canRetryWithRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
{#if canOverride}

{detail.safetyOverrideUnavailableReason ?? 'Confirm to bypass player safety checks for this operation.'}

{/if} - {#if detail.canResumeVerification}{/if} + {#if detail.canResumeVerification}{/if} + {#if detail.canRetryWithRebuild}

Starts a new rebuild for the same lobby and port. Deletes instance files and uses the latest promoted setup.

{/if} {#if detail.canResumeRebuild}{/if} {#if detail.canResumeCleanup}{/if} {#if detail.canCancel}

Stops remaining work and health checks. Changes already made are not undone.

{/if} diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index b47e0b1..478f5d8 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -1503,6 +1503,169 @@ describe('automatic capacity admission and placement', () => { }); }); +describe('native verification resume and explicit failed-creation retry', () => { + async function failedCreation(state: 'failed' | 'interrupted' = 'failed') { + const job = await queue(); await advance(); + const original = (await operationFor(job.deploymentId))!; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, original.id, { state: 'failed', error: 'Host runtime preflight failed' }); + await advance(); + if (state === 'interrupted') { + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId); + } + return { job, original }; + } + + it.each(['failed', 'interrupted'] as const)('retries a %s creation through a new rebuild while retaining the original failure', async state => { + const { job, original } = await failedCreation(state); + const history = await import('../../src/lib/server/provisioning/history.js'); + expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: true }); + const { POST: resume } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js'); + expect(resume({ params: { id: job.deploymentId } }).status).toBe(409); + const { POST: retry } = await import('../../src/routes/api/provisioning/[id]/retry-rebuild/+server.js'); + const response = await retry({ params: { id: job.deploymentId } }); + expect(response.status).toBe(202); + const next = await response.json() as { ok: boolean; deploymentId: string }; + expect(next.ok).toBe(true); + expect(next.deploymentId).not.toBe(job.deploymentId); + expect(await detail(next.deploymentId)).toMatchObject({ action: 'rebuild', state: 'waiting', server: job.name }); + expect(history.getDeployment(job.deploymentId)).toMatchObject({ + canRetryWithRebuild: false, + deployment: { state, error: 'Host runtime preflight failed' }, + progress: { recovery: { state: 'transferred', operationId: next.deploymentId } } + }); + expect((await retry({ params: { id: job.deploymentId } })).status).toBe(409); + const { db } = await import('../../src/lib/server/db/index.js'); + expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server=?').get(job.name)).toEqual({ count: 2 }); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)).toMatchObject({ id: original.instanceId, generation: 1, port: original.payload.port }); + remote.state = 'missing'; await advance(); + const rebuilding = (await operationFor(next.deploymentId))!; + expect(rebuilding).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: original.instanceId, + payload: { name: job.name, port: original.payload.port, nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } }); + expect(rebuilding.id).not.toBe(original.id); + expect(nativeInstance(job.name)?.generation).toBe(1); + remote.state = 'running'; await acknowledge(next.deploymentId); + expect((await detail(next.deploymentId)).state).toBe('done'); + expect(nativeInstance(job.name)?.generation).toBe(2); + // An interrupted original is reconciled to its already-failed receipt + // on the next worker pass; the failure itself is never cleared. + expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: 'Host runtime preflight failed' }); + expect(await operationFor(job.deploymentId)).toMatchObject({ id: original.id, state: 'failed' }); + expect(db().prepare("SELECT kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.rebuild','instance.restart') ORDER BY rowid").all()) + .toEqual([{ kind: 'instance.create' }, { kind: 'instance.rebuild' }]); + }); + + it.each([ + ['create', false], ['create', true], ['rebuild', false], ['rebuild', true], + ['recovery-restart', false], ['recovery-restart', true], ['recovery-rebuild', false], ['recovery-rebuild', true] + ] as const)('resumes completed %s verification with the same command (receipt applied=%s)', async (action, applied) => { + let job = await queue(); await advance(); + if (action === 'rebuild') { + await acknowledge(job.deploymentId); + const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); + const rebuild = queueNativeRebuild(job.name, 'test'); + if (!rebuild.ok) throw new Error(rebuild.message); + job = rebuild; await advance(); + } else if (action.startsWith('recovery')) { + remote.state = 'unhealthy'; await acknowledge(job.deploymentId); + if (action === 'recovery-rebuild') await acknowledge(job.deploymentId); + } + const operation = (await operationFor(job.deploymentId))!; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + remote.state = 'starting'; reportHostOperation(hostId, operation.id, { state: 'completed' }); + if (applied) await advance(); + const history = await import('../../src/lib/server/provisioning/history.js'); + history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted for inspection' }); + expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: true, canRetryWithRebuild: false }); + const { db } = await import('../../src/lib/server/db/index.js'); + const commands = db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all(); + const { POST } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js'); + expect(POST({ params: { id: job.deploymentId } }).status).toBe(202); + remote.state = 'running'; await advance(); + expect((await detail(job.deploymentId)).state).toBe('done'); + expect((await operationFor(job.deploymentId))!.id).toBe(operation.id); + expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all()).toEqual(commands); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)?.generation).toBe(operation.generation! + (operation.kind === 'instance.rebuild' ? 1 : 0)); + expect(history.getDeployment(job.deploymentId)!.events).toContainEqual(expect.objectContaining({ + kind: 'outcome', message: 'Failed: Verification interrupted for inspection' + })); + }); + + it.each(['pending', 'accepted', 'running', 'unknown', 'failed'] as const)('does not offer verification of a %s command or replay an ambiguous result', async state => { + const job = await queue(); await advance(); + const operation = (await operationFor(job.deploymentId))!; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + if (state !== 'pending') reportHostOperation(hostId, operation.id, { state }); + const history = await import('../../src/lib/server/provisioning/history.js'); + history.finishDeployment(job.deploymentId, { ok: false, message: 'Controller observation interrupted' }); + expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: state === 'failed' }); + expect(history.resumeVerification(job.deploymentId)).toBe(false); + if (state !== 'failed') { + const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js'); + expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 }); + } + const { db } = await import('../../src/lib/server/db/index.js'); + if (state === 'unknown') db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId); + await advance(); + expect((await operationFor(job.deploymentId))!.id).toBe(operation.id); + expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").all()) + .toEqual([{ id: operation.id }]); + }); + + it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work', 'creation cleanup', 'recovery attempt']) + ('refuses to retry a failed creation after its %s changes', async change => { + const { job, original } = await failedCreation(); + const { db } = await import('../../src/lib/server/db/index.js'); + if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name); + if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id); + if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id); + if (change === 'command host') { + const { createHost } = await import('../../src/lib/server/hosts/index.js'); + db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id); + } + if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name); + if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name); + if (change === 'newer work') { + const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); + beginDeployment(job.name, 'restart'); + } + if (change === 'creation cleanup') { + const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String)); + } + if (change === 'recovery attempt') db().prepare("UPDATE native_provisioning SET recovery_step='restart' WHERE deployment_id=?").run(job.deploymentId); + const history = await import('../../src/lib/server/provisioning/history.js'); + const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js'); + expect(history.getDeployment(job.deploymentId)?.canRetryWithRebuild).toBe(false); + expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 }); + expect(db().prepare("SELECT id FROM server_deployments WHERE action='rebuild'").all()).toEqual([]); + }); + + it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work']) + ('refuses to resume a completed creation after its %s changes', async change => { + const job = await queue(); await advance(); + const original = (await operationFor(job.deploymentId))!; + const { reportHostOperation, createHost } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, original.id, { state: 'completed' }); + const history = await import('../../src/lib/server/provisioning/history.js'); + history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted' }); + const { db } = await import('../../src/lib/server/db/index.js'); + if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name); + if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id); + if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id); + if (change === 'command host') db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id); + if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name); + if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name); + if (change === 'newer work') history.beginDeployment(job.name, 'restart'); + expect(history.getDeployment(job.deploymentId)?.canResumeVerification).toBe(false); + expect(history.resumeVerification(job.deploymentId)).toBe(false); + expect((await operationFor(job.deploymentId))!.id).toBe(original.id); + }); +}); + describe('unhealthy recovery within native create and rebuild operations', () => { async function launch(action: 'create' | 'rebuild') { const job = await queue(); await advance(); diff --git a/tests/unit/provisioning-presentation.test.ts b/tests/unit/provisioning-presentation.test.ts index 84b396c..0be27b3 100644 --- a/tests/unit/provisioning-presentation.test.ts +++ b/tests/unit/provisioning-presentation.test.ts @@ -19,6 +19,20 @@ function fixture(): ProvisioningDetail { } describe('provisioning outcome and evidence presentation', () => { + it('distinguishes a failed creation retry from readiness verification and its linked recovery', () => { + const detail = fixture(); + detail.deployment.state = 'failed'; + detail.canRetryWithRebuild = true; + expect(provisioningOutcome(detail)).toMatchObject({ active: false, headline: 'Create failed' }); + expect(provisioningOutcome(detail).description).toContain('new attempt for the same lobby and port'); + detail.canRetryWithRebuild = false; + detail.progress = progress({ recovery: { state: 'transferred', operationId: 'recovery-a', at, message: 'Rebuild requested' } }); + expect(provisioningOutcome(detail).headline).toBe('Create failed'); + expect(provisioningOutcome(detail).description).toContain('linked operation'); + detail.progress = progress(); + detail.canResumeVerification = true; + expect(provisioningOutcome(detail).description).toContain('remaining restart and rebuild recovery steps'); + }); it('distinguishes cancellation from confirmed cleanup and validates its link data', () => { const detail = fixture(); detail.deployment.state = 'cancelled'; @@ -142,6 +156,15 @@ describe('provisioning outcome and evidence presentation', () => { }); describe('provisioning response and return-context boundaries', () => { + it('accepts optional retry eligibility only as an authoritative boolean', () => { + const detail = fixture(); + for (const canRetryWithRebuild of [undefined, false, true]) { + expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(true); + } + for (const canRetryWithRebuild of ['true', 1, null]) { + expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(false); + } + }); it('accepts persisted usernames and legacy details but rejects malformed requesters', () => { const detail = fixture(); for (const requested_by of [undefined, null, 'alice']) { diff --git a/tests/unit/provisioning-refresh.test.ts b/tests/unit/provisioning-refresh.test.ts index eef46d3..c87cba1 100644 --- a/tests/unit/provisioning-refresh.test.ts +++ b/tests/unit/provisioning-refresh.test.ts @@ -23,6 +23,15 @@ function deferred() { } describe('provisioning detail refresh ownership', () => { + it('requests a rebuild retry and refreshes the original failed operation', async () => { + const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true, deploymentId: 'recovery-a' })).mockResolvedValueOnce(response('a', 2)); + const page = setup(fetcher); + page.setOperation('a', detail('a')); + await page.act('retry-rebuild', 'Follow the linked recovery operation.'); + expect(fetcher).toHaveBeenNthCalledWith(1, '/api/provisioning/a/retry-rebuild', expect.objectContaining({ method: 'POST' })); + expect(fetcher).toHaveBeenNthCalledWith(2, '/api/provisioning/a?events=recent', expect.anything()); + expect(page.state()).toMatchObject({ id: 'a', detail: detail('a', 2), notice: 'Follow the linked recovery operation.', busy: null }); + }); it('sends explicit override confirmation and refreshes the operation after acceptance', async () => { const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true })).mockResolvedValueOnce(response('a', 2)); const page = setup(fetcher);