Offer rebuild retry for failed creation commands
This commit is contained in:
@@ -101,6 +101,7 @@ export interface ProvisioningDetail {
|
||||
safetyOverride?: { actorId: string; actorName: string; confirmedAt: string } | null;
|
||||
canCancel?: boolean;
|
||||
canCancelCreation?: boolean;
|
||||
canRetryWithRebuild?: boolean;
|
||||
cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null;
|
||||
backend?: 'native';
|
||||
workerActivity?: WorkerActivity[];
|
||||
|
||||
@@ -14,6 +14,7 @@ import { canCancelCreation } from './cancel-creation.js';
|
||||
import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js';
|
||||
import { committedRebalanceStartup } from './concurrency.js';
|
||||
import { nativeDeployment } from '../backends/native.js';
|
||||
import { ensureHostSchema } from '../hosts/schema.js';
|
||||
import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
|
||||
import { serverTitle } from '../../ui/server-labels.js';
|
||||
import { provisioningQueueStatus } from '../../ui/provisioning-queue.js';
|
||||
@@ -149,7 +150,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
|
||||
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
|
||||
recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null,
|
||||
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
|
||||
canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
|
||||
canResumeRebuild: canResumeRebuild(id), canRetryWithRebuild: canRetryWithRebuild(id),
|
||||
canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
|
||||
progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
|
||||
warningMessages: warningMessages(deployment.warnings), generatedAt };
|
||||
})();
|
||||
@@ -197,17 +199,43 @@ export function resumeCleanup(id: string): boolean {
|
||||
}
|
||||
|
||||
export function canResumeVerification(id: string): boolean {
|
||||
ensureHostSchema(db());
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
||||
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
||||
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
||||
WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND (EXISTS (
|
||||
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL) OR EXISTS (
|
||||
SELECT 1 FROM ec_host_migrations m JOIN native_provisioning n ON n.deployment_id=m.deployment_id
|
||||
WHERE m.deployment_id=d.id AND m.removed_at IS NOT NULL AND n.operation_id IS NOT NULL)))) AND d.state IN ('failed', 'interrupted')
|
||||
JOIN servers s ON s.name = j.server AND s.game_port = j.port AND s.destroyed_at IS NULL
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
LEFT JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
|
||||
LEFT JOIN ec_host_operations o ON o.id=n.operation_id
|
||||
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('create','rebuild')
|
||||
AND d.state IN ('failed', 'interrupted')
|
||||
AND ((n.deployment_id IS NOT NULL AND i.port=j.port AND o.state='completed'
|
||||
AND o.instance_id=i.id AND o.host_id=i.host_id AND o.generation=n.generation
|
||||
AND ((o.kind IN ('instance.create','instance.restart') AND i.generation=n.generation)
|
||||
OR (o.kind='instance.rebuild' AND i.generation IN (n.generation,n.generation+1))))
|
||||
OR (n.deployment_id IS NULL AND (d.action='create' OR EXISTS (
|
||||
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id=d.id AND r.removed_at IS NOT NULL))))
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
||||
.get(id);
|
||||
}
|
||||
|
||||
/** A conclusively failed initial create can enter the normal rebuild flow. */
|
||||
export function canRetryWithRebuild(id: string): boolean {
|
||||
ensureHostSchema(db());
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
||||
JOIN server_creation_jobs j ON j.deployment_id=d.id
|
||||
JOIN servers s ON s.name=d.server AND s.game_port=j.port AND s.destroyed_at IS NULL
|
||||
JOIN native_provisioning n ON n.deployment_id=d.id AND n.recovery_step IS NULL
|
||||
JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
|
||||
AND i.generation=n.generation AND i.port=j.port
|
||||
JOIN ec_host_operations o ON o.id=n.operation_id AND o.instance_id=i.id
|
||||
AND o.host_id=i.host_id AND o.generation=n.generation
|
||||
WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action='create'
|
||||
AND d.state IN ('failed','interrupted') AND o.kind='instance.create' AND o.state='failed'
|
||||
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)`)
|
||||
.get(id);
|
||||
}
|
||||
|
||||
export function canResumeRebuild(id: string): boolean {
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
|
||||
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
|
||||
@@ -226,7 +254,7 @@ export function resumeRebuild(id: string): boolean {
|
||||
})();
|
||||
}
|
||||
|
||||
/** Resume observation only. Never repeat a container creation or restart. */
|
||||
/** Resume readiness after a completed command; unhealthy recovery still applies. */
|
||||
export function resumeVerification(id: string): boolean {
|
||||
return db().transaction(() => {
|
||||
if (!canResumeVerification(id)) return false;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Request/observation boundary used by APIs, campaigns, beta tests, and schedules.
|
||||
// Execution is driven exclusively by the provisioning worker.
|
||||
export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
|
||||
export { requestLifecycle, requestRetryWithRebuild, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
|
||||
export type { LifecycleOperation, RestartSource } from './requests.js';
|
||||
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextLobbyNumber } from './instances.js';
|
||||
export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';
|
||||
|
||||
@@ -3,7 +3,7 @@ import { currentRequestUsername } from '../auth/request-context.js';
|
||||
import { maintenanceAllowed } from '../maintenance/policies.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
|
||||
import { beginDeployment, canRetryWithRebuild, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
|
||||
import { queueRebuildServer } from './instances.js';
|
||||
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
@@ -166,6 +166,23 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
|
||||
|
||||
type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string };
|
||||
|
||||
/** Keep the failed launch immutable and let normal rebuild admission own recovery. */
|
||||
export function requestRetryWithRebuild(id: string): LifecycleRequestResult {
|
||||
return db().transaction(() => {
|
||||
if (!canRetryWithRebuild(id)) return { ok: false as const, status: 409,
|
||||
error: 'This failed creation can no longer be retried with a rebuild. Refresh and inspect its current operation.' };
|
||||
const original = provisioningOperation(id)!;
|
||||
const result = requestLifecycle(original.server, 'rebuild');
|
||||
if (!result.ok) return result;
|
||||
recordProgress(id, { recovery: { state: 'transferred', operationId: result.deploymentId,
|
||||
message: 'A new rebuild was requested for this server. The failed creation and its error are preserved.' },
|
||||
event: { kind: 'retry-requested', message: 'Retry with rebuild requested', details: { operationId: result.deploymentId } } });
|
||||
recordProgress(result.deploymentId, { event: { kind: 'retry-requested',
|
||||
message: 'Rebuild requested after failed creation', details: { previousOperationId: id } } });
|
||||
return result;
|
||||
})();
|
||||
}
|
||||
|
||||
function watchdogRequestGuard(server: string, incidentId: string): Extract<LifecycleRequestResult, { ok: false }> | null {
|
||||
const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined;
|
||||
if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' };
|
||||
|
||||
@@ -76,11 +76,13 @@ export function provisioningOutcome(detail: ProvisioningDetail) {
|
||||
let description: string;
|
||||
if (detail.canResumeCleanup) description = 'The remote deployment was deleted. Resume the remaining local cleanup without repeating container removal.';
|
||||
else if (detail.canResumeRebuild) description = 'The remote deployment was deleted. Resume local cleanup, then create a fresh instance in the reserved server slot.';
|
||||
else if (detail.canResumeVerification) description = 'Resume observation of the existing instance. This does not create, restart, or rebuild a container.';
|
||||
else if (detail.canResumeVerification) description = 'Resume readiness checks for the existing instance. An unhealthy container may continue through this operation’s remaining restart and rebuild recovery steps.';
|
||||
else if (detail.canRetryWithRebuild) description = 'The host reported that creation failed. Retry with rebuild starts a new attempt for the same lobby and port using the latest promoted setup.';
|
||||
else if (detail.cancellation) description = detail.cancellation.state === 'done'
|
||||
? 'Creation was cancelled. Its instance files, container, and port reservations have been removed.'
|
||||
: 'Creation was cancelled. Cleanup is tracked separately below; ports remain reserved until the host confirms removal.';
|
||||
else if (operation.state === 'cancelled') description = 'This request is cancelled. Recorded prior work is preserved below; remaining work will not run.';
|
||||
else if (failure && progress?.recovery?.state === 'transferred' && progress.recovery.operationId && progress.recovery.operationId !== operation.id) description = 'A recovery operation has been requested for this server. Follow the linked operation below; this record keeps the original failure.';
|
||||
else if (failure) description = progress?.command?.state === 'uncertain'
|
||||
? 'The remote response was uncertain. Inspect the recorded result and server before requesting another operation.'
|
||||
: 'Review the recorded failure and execution evidence before submitting another operation.';
|
||||
@@ -162,6 +164,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi
|
||||
if (!record(value) || !record(value.deployment)) return false;
|
||||
if (value.canCancel !== undefined && typeof value.canCancel !== 'boolean') return false;
|
||||
if (value.canCancelCreation !== undefined && typeof value.canCancelCreation !== 'boolean') return false;
|
||||
if (value.canRetryWithRebuild !== undefined && typeof value.canRetryWithRebuild !== 'boolean') return false;
|
||||
if (value.cancellation !== undefined && value.cancellation !== null && !(record(value.cancellation)
|
||||
&& text(value.cancellation.cleanupId) && text(value.cancellation.state) && text(value.cancellation.stage)
|
||||
&& nullableText(value.cancellation.error))) return false;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export type ProvisioningMutation = 'resume' | 'cancel' | 'cancel-creation' | 'override';
|
||||
export type ProvisioningMutation = 'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override';
|
||||
|
||||
export interface ProvisioningRefreshState<T> {
|
||||
id: string;
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { requestRetryWithRebuild } from '$lib/server/provisioning/index.js';
|
||||
import { kickProvisioning } from '$lib/server/pipeline/worker.js';
|
||||
|
||||
export function POST({ params }: { params: { id: string } }) {
|
||||
const result = requestRetryWithRebuild(params.id);
|
||||
if (!result.ok) return json({ error: result.error }, { status: result.status });
|
||||
void kickProvisioning().catch(err => console.error('[provisioning] rebuild retry kick failed:', err));
|
||||
return json(result, { status: 202 });
|
||||
}
|
||||
@@ -59,6 +59,17 @@
|
||||
});
|
||||
onMount(() => pollVisible(() => { now = Date.now(); return controller.refresh(); }, 5000, false));
|
||||
onDestroy(() => controller.dispose());
|
||||
async function retryWithRebuild() {
|
||||
if (!deployment || !detail?.canRetryWithRebuild || view.busy) return;
|
||||
const id = deployment.id;
|
||||
const confirmed = await confirmAction({
|
||||
title: 'Retry creation with a rebuild?',
|
||||
message: `Rebuild ${serverTitle(deployment.server)} (${deployment.server}) using the latest promoted setup. This keeps the same lobby and port, deletes its container and all instance files, and starts a new provisioning operation.\n\nPlayer safety, capacity limits, and protected work checks still apply. The original failure remains in this operation’s history.`,
|
||||
confirmLabel: 'Retry with rebuild', danger: true
|
||||
});
|
||||
if (confirmed && view.id === id && detail?.canRetryWithRebuild) await controller.act('retry-rebuild',
|
||||
'Rebuild retry requested. Follow the linked recovery operation below for progress.');
|
||||
}
|
||||
async function overrideSafety() {
|
||||
if (!deployment || !canOverride || view.busy || detail?.safetyOverrideUnavailableReason) return;
|
||||
const id = deployment.id;
|
||||
@@ -107,10 +118,11 @@
|
||||
{#if detail.warningMessages.length}<div class="warnings"><h3>Recorded warnings</h3><ul>{#each detail.warningMessages as warning}<li>{warning}</li>{/each}</ul></div>{/if}
|
||||
|
||||
{#if detail.safetyOverride}<div class="warnings"><h3>Admin override enabled</h3><p>Player safety checks bypassed for this operation. Confirmed by {detail.safetyOverride.actorName} · <RelativeTime value={detail.safetyOverride.confirmedAt} />.</p></div>{/if}
|
||||
{#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
|
||||
{#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canRetryWithRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
|
||||
<div class="recovery-actions">
|
||||
{#if canOverride}<div><Button variant="danger" disabled={!!view.busy || !!detail.safetyOverrideUnavailableReason} loading={view.busy === 'override'} onclick={overrideSafety}>Admin override…</Button><p class="action-help">{detail.safetyOverrideUnavailableReason ?? 'Confirm to bypass player safety checks for this operation.'}</p></div>{/if}
|
||||
{#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. The existing instance will be observed without another creation or restart.')}>Resume health verification</Button>{/if}
|
||||
{#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. An unhealthy container may continue through this operation’s remaining recovery steps.')}>Resume health verification</Button>{/if}
|
||||
{#if detail.canRetryWithRebuild}<div><Button variant="primary" loading={view.busy === 'retry-rebuild'} disabled={!!view.busy} onclick={retryWithRebuild}>Retry with rebuild</Button><p class="action-help">Starts a new rebuild for the same lobby and port. Deletes instance files and uses the latest promoted setup.</p></div>{/if}
|
||||
{#if detail.canResumeRebuild}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Rebuild cleanup requested. Fresh creation waits for local cleanup to finish.')}>Resume rebuild cleanup</Button>{/if}
|
||||
{#if detail.canResumeCleanup}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Local cleanup requested. Remote container deletion will not be repeated.')}>Resume cleanup</Button>{/if}
|
||||
{#if detail.canCancel}<div><Button disabled={!!view.busy} loading={view.busy === 'cancel'} onclick={() => controller.act('cancel', 'Operation cancelled. Already-submitted commands may finish; no further steps will be started.')}>Cancel operation</Button><p class="action-help">Stops remaining work and health checks. Changes already made are not undone.</p></div>{/if}
|
||||
|
||||
Reference in New Issue
Block a user