Complete provisioning job detail pages and lifecycle evidence

This commit is contained in:
2026-09-28 14:17:19 -04:00
parent 5b65520bbc
commit 3fb59cd598
41 changed files with 2866 additions and 346 deletions
+121
View File
@@ -0,0 +1,121 @@
import type { ProvisioningAction } from './provisioning.js';
import type { LaunchRecipeSummary } from './deployment-plan.js';
export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'prepare', 'create-deployment',
'container-action', 'remove-container', 'delete-deployment', 'cleanup', 'verify-container', 'verify-game',
'verify-map', 'verify-stopped'] as const;
export type ProvisioningPhase = typeof provisioningPhases[number];
export type EvidenceResult = 'pass' | 'waiting' | 'fail' | 'unknown';
export type EvidenceSource = 'komodo' | 'rcon' | 'telemetry' | 'combined' | null;
export interface ProvisioningCheck {
key: 'container' | 'game' | 'map' | 'occupancy' | 'stopped';
label: string;
result: EvidenceResult;
source: EvidenceSource;
checkedAt: string;
receivedAt: string | null;
expected: string | number | null;
actual: string | number | null;
message: string | null;
}
export interface ProvisioningBlocker {
code: string;
message: string;
automatic: boolean;
href?: string | null;
label?: string | null;
}
export interface ProvisioningCheckpoint {
key: string;
label: string;
at: string;
phase: ProvisioningPhase | null;
}
export interface RecoveryObservation {
state: 'held' | 'released' | 'transferred' | 'unknown';
at: string;
message: string;
operationId?: string | null;
}
export interface ProvisioningProgress {
version: 1;
phase: ProvisioningPhase | null;
phaseStartedAt: string | null;
observedAt: string | null;
blocker: ProvisioningBlocker | null;
checks: ProvisioningCheck[];
checkpoints: ProvisioningCheckpoint[];
command: { state: 'attempted' | 'accepted' | 'rejected' | 'uncertain'; at: string; message: string } | null;
recovery: RecoveryObservation | null;
recipeStatus: 'queued' | 'attempted' | null;
}
export interface ProvisioningEvent {
id: number;
created_at: string;
message: string;
kind?: string | null;
phase_key?: string | null;
details_json?: string | null;
}
export interface ProvisioningDeployment {
id: string;
server: string;
server_deleted_at: string | null;
action: ProvisioningAction;
state: string;
stage: string;
started_at: string;
finished_at: string | null;
submitted_at?: string | null;
checked_at?: string | null;
players?: number | null;
triggered_by: string;
error: string | null;
warnings: string;
server_exists: number;
expected_map?: string | null;
}
export interface ProvisioningSource {
kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog';
source_id: string;
server?: string;
label: string;
href: string | null;
}
export interface ProvisioningCapacity {
evaluatedAt: string;
pools: {
kind: 'startup' | 'restart';
limit: number;
used: number;
holders: { id: string | null; server: string }[];
requestOrder: number | null;
ownsSlot: boolean;
}[];
}
export interface ProvisioningDetail {
deployment: ProvisioningDeployment;
events: ProvisioningEvent[];
activity: { total: number; latestId: number | null; failureExcerpt: ProvisioningEvent | null };
sources: ProvisioningSource[];
recipe: LaunchRecipeSummary | null;
blueprintName: string | null;
canResumeVerification: boolean;
canResumeCleanup: boolean;
canResumeRebuild: boolean;
canCancelRemoval: boolean;
collection: { id: string; name: string } | null;
progress: ProvisioningProgress | null;
capacity: ProvisioningCapacity;
warningMessages: string[];
generatedAt: string;
}
export interface ProvisioningEventPage {
events: ProvisioningEvent[];
total: number;
totalEvents: number;
latestId: number | null;
watermark: number;
nextCursor: number | null;
}
+14 -3
View File
@@ -243,8 +243,10 @@ function migrateProvisioningActions(handle: Database.Database): void {
handle.transaction(() => {
handle.exec(table.sql.replace('server_deployments', 'server_deployments_new')
.replace("'create', 'destroy'", "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"));
handle.exec(`INSERT INTO server_deployments_new (rowid, id, server, action, state, stage, started_at, finished_at, triggered_by, error, warnings)
SELECT rowid, id, server, action, state, stage, started_at, finished_at, triggered_by, error, warnings FROM server_deployments;
const columns = (handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[])
.map(column => `"${column.name.replaceAll('"', '""')}"`).join(', ');
handle.exec(`INSERT INTO server_deployments_new (rowid, ${columns})
SELECT rowid, ${columns} FROM server_deployments;
DROP TABLE server_deployments;
ALTER TABLE server_deployments_new RENAME TO server_deployments;`);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Provisioning migration found broken references');
@@ -309,7 +311,16 @@ export function db(): Database.Database {
migrateTable(handle, 'server_deployments', [
{ name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
{ name: 'checked_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT' },
{ name: 'players', ddl: 'ALTER TABLE server_deployments ADD COLUMN players INTEGER' }
{ name: 'players', ddl: 'ALTER TABLE server_deployments ADD COLUMN players INTEGER' },
{ name: 'phase_key', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_key TEXT' },
{ name: 'phase_started_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT' },
{ name: 'observed_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT' },
{ name: 'observation_json', ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT' }
]);
migrateTable(handle, 'server_deployment_events', [
{ name: 'kind', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN kind TEXT' },
{ name: 'phase_key', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT' },
{ name: 'details_json', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT' }
]);
for (const table of ['deployment_servers', 'beta_servers', 'maintenance_jobs']) {
migrateTable(handle, table, [{ name: 'provisioning_id', ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` }]);
+9 -2
View File
@@ -452,14 +452,21 @@ CREATE TABLE IF NOT EXISTS server_deployments (
finished_at TEXT,
triggered_by TEXT NOT NULL,
error TEXT,
warnings TEXT NOT NULL DEFAULT '[]'
warnings TEXT NOT NULL DEFAULT '[]',
phase_key TEXT,
phase_started_at TEXT,
observed_at TEXT,
observation_json TEXT
);
CREATE TABLE IF NOT EXISTS server_deployment_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
deployment_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
created_at TEXT NOT NULL,
message TEXT NOT NULL
message TEXT NOT NULL,
kind TEXT,
phase_key TEXT,
details_json TEXT
);
CREATE INDEX IF NOT EXISTS idx_server_deployment_events ON server_deployment_events(deployment_id, id);
+27 -9
View File
@@ -36,30 +36,48 @@ export function telemetryEvidence(server: string, after: string | null = latestG
/** Positive occupancy from either source always vetoes an empty-server claim. */
export async function probePlayerCount(server: string): Promise<number | null> {
return (await probePlayerEvidence(server)).players;
}
export interface PlayerEvidence {
players: number | null;
source: 'rcon' | 'telemetry' | null;
observedAt: string;
receivedAt?: string;
}
/** The same eligibility decision, with provenance from the probes already performed. */
export async function probePlayerEvidence(server: string): Promise<PlayerEvidence> {
const fromTelemetry = (value: TelemetryEvidence): PlayerEvidence => ({
players: value.players, source: 'telemetry', observedAt: new Date().toISOString(), receivedAt: value.receivedAt
});
const occupied = telemetryEvidence(server);
if (occupied && occupied.players > 0) return occupied.players;
if (occupied && occupied.players > 0) return fromTelemetry(occupied);
const players = await rconGetPlayerCount(server).catch(() => null);
// Read after the network request: a player may join while RCON is pending.
const telemetry = telemetryEvidence(server);
if (players !== null && players > 0) return players;
if (telemetry && telemetry.players > 0) return telemetry.players;
return players === 0 || telemetry?.players === 0 ? 0 : null;
if (players !== null && players > 0) return { players, source: 'rcon', observedAt: new Date().toISOString() };
if (telemetry && telemetry.players > 0) return fromTelemetry(telemetry);
if (players === 0) return { players, source: 'rcon', observedAt: new Date().toISOString() };
if (telemetry?.players === 0) return fromTelemetry(telemetry);
return { players: null, source: null, observedAt: new Date().toISOString() };
}
/** Komodo readiness remains a caller prerequisite. Old-process telemetry is excluded. */
export async function probeServerStatus(server: string, after?: string | null): Promise<
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry' } | { ok: false; reason: string }
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry'; observedAt: string; receivedAt?: string } | { ok: false; reason: string; observedAt: string }
> {
const current = telemetryEvidence(server, after);
if (current) return { ok: true, ...current, source: 'telemetry' };
if (current) return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() };
let reason = 'RCON unavailable';
try {
const rcon = await rconStatusResult(server);
const latest = telemetryEvidence(server, after);
if (latest) return { ok: true, ...latest, source: 'telemetry' };
if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon' };
if (latest) return { ok: true, ...latest, source: 'telemetry', observedAt: new Date().toISOString() };
if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', observedAt: new Date().toISOString() };
reason = rcon.reason;
} catch (err) { reason = (err as Error).message; }
const telemetry = telemetryEvidence(server, after);
return telemetry ? { ok: true, ...telemetry, source: 'telemetry' } : { ok: false, reason };
return telemetry ? { ok: true, ...telemetry, source: 'telemetry', observedAt: new Date().toISOString() }
: { ok: false, reason, observedAt: new Date().toISOString() };
}
+56
View File
@@ -0,0 +1,56 @@
import { db } from '../db/index.js';
import type { ProvisioningEvent, ProvisioningEventPage } from '../../provisioning-detail.js';
type ActivityRow = ProvisioningEvent;
export interface ActivityQuery {
order?: 'newest' | 'oldest';
query?: string;
cursor?: number;
watermark?: number;
limit?: number;
}
export function activitySummary(id: string): { total: number; latestId: number | null } {
return db().prepare(`SELECT COUNT(*) AS total, MAX(id) AS latestId
FROM server_deployment_events WHERE deployment_id = ?`).get(id) as { total: number; latestId: number | null };
}
/** ID ordering and a fixed upper watermark keep older pages stable as new events arrive. */
export function deploymentActivity(id: string, options: ActivityQuery = {}): ProvisioningEventPage {
const summary = activitySummary(id);
const watermark = Math.min(options.watermark ?? summary.latestId ?? 0, summary.latestId ?? 0);
const limit = Math.max(1, Math.min(100, Math.trunc(options.limit ?? 25)));
const newest = options.order !== 'oldest';
const query = options.query ?? '';
const base = 'deployment_id = ? AND id <= ? AND instr(lower(message), lower(?)) > 0';
const values: (string | number)[] = [id, watermark, query];
const total = (db().prepare(`SELECT COUNT(*) AS n FROM server_deployment_events WHERE ${base}`)
.get(...values) as { n: number }).n;
const cursor = options.cursor === undefined ? '' : ` AND id ${newest ? '<' : '>'} ?`;
if (options.cursor !== undefined) values.push(options.cursor);
const rows = db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json
FROM server_deployment_events WHERE ${base}${cursor}
ORDER BY id ${newest ? 'DESC' : 'ASC'} LIMIT ?`).all(...values, limit + 1) as ActivityRow[];
const more = rows.length > limit;
const events = rows.slice(0, limit);
return { events, total, totalEvents: summary.total, latestId: summary.latestId, watermark,
nextCursor: more ? events.at(-1)!.id : null };
}
export function deploymentEvents(id: string, recent = false): ProvisioningEvent[] {
if (recent) return (db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json
FROM server_deployment_events WHERE deployment_id = ? ORDER BY id DESC LIMIT 5`).all(id) as ActivityRow[]).reverse();
return (db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json
FROM server_deployment_events WHERE deployment_id = ? ORDER BY id`).all(id) as ActivityRow[]);
}
export function latestFailureEvent(id: string): ProvisioningEvent | null {
const row = db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json
FROM server_deployment_events WHERE deployment_id = ?
AND (kind IN ('failed', 'interrupted') OR (kind = 'outcome'
AND CASE WHEN json_valid(details_json) THEN json_extract(details_json, '$.state') END IN ('failed', 'interrupted'))
OR (kind IS NULL AND (message LIKE 'Failed:%' OR message LIKE 'Interrupted by application restart.%')))
ORDER BY id DESC LIMIT 1`).get(id) as ActivityRow | undefined;
return row ?? null;
}
+152 -39
View File
@@ -2,20 +2,35 @@ import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { recipeForDeployment, summarizeRecipe } from '../fleet/launch-recipes.js';
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js';
import { fleetSettings } from '../settings.js';
import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
const id = randomUUID();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
recordDeploymentEvent(id, action === 'create' ? 'Server creation requested'
: action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested`);
db().transaction(() => {
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested',
label: action === 'create' ? 'Server creation requested'
: action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested` } });
})();
return id;
}
export function deploymentStage(id: string, stage: string): void {
const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
if (previous && previous.stage !== stage) recordDeploymentEvent(id, stage);
export function deploymentStage(id: string, stage: string, update?: ProgressUpdate): void {
db().transaction(() => {
const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
const changed = !!previous && previous.stage !== stage;
const before = update && changed ? activitySummary(id).latestId ?? 0 : 0;
db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
if (update) recordProgress(id, update);
// Structured transitions may already carry the exact legacy stage text.
const alreadyRecorded = update && changed && db().prepare(`SELECT 1 FROM server_deployment_events
WHERE deployment_id = ? AND id > ? AND message = ? LIMIT 1`).get(id, before, stage);
if (changed && !alreadyRecorded) recordDeploymentEvent(id, stage);
})();
}
export function recordDeploymentEvent(id: string, message: string): void {
@@ -23,19 +38,83 @@ export function recordDeploymentEvent(id: string, message: string): void {
.run(id, new Date().toISOString(), message);
}
export function getDeployment(id: string) {
const deployment = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id);
if (!deployment) return null;
const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
const sources = db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id').all(id);
const launch = recipeForDeployment(id);
const collection = db().prepare(`SELECT c.id, c.name FROM (
SELECT collection_id, deployment_id FROM server_collection_members
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
return { deployment, events, sources, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection };
function warningMessages(raw: string): string[] {
try {
const parsed: unknown = JSON.parse(raw);
if (!Array.isArray(parsed)) return ['Recorded warnings could not be read.'];
const messages = parsed.filter((value): value is string => typeof value === 'string');
if (messages.length !== parsed.length) messages.push('Some recorded warnings could not be read.');
return messages;
} catch { return ['Recorded warnings could not be read.']; }
}
function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
const id = source.source_id;
if (source.kind === 'campaign') {
const row = db().prepare('SELECT action FROM deployments WHERE id = ?').get(id) as { action: string } | undefined;
const action = row?.action === 'promote' ? 'Promotion' : row?.action === 'rollback' ? 'Rollback' : row?.action === 'drift' ? 'Reconciliation' : 'Campaign';
return { ...source, label: `${action} ${id.slice(0, 8)}`, href: row ? `/campaigns/${encodeURIComponent(id)}` : null };
}
if (source.kind === 'beta') {
const row = db().prepare('SELECT notes FROM betas WHERE id = ?').get(id) as { notes: string | null } | undefined;
const label = row?.notes?.trim().split('\n')[0].slice(0, 120);
return { ...source, label: label ? `Beta #${id}: ${label}` : `Beta #${id}`, href: row ? `/betas/${encodeURIComponent(id)}` : null };
}
if (source.kind === 'watchdog') {
const row = db().prepare('SELECT summary FROM watchdog_incidents WHERE id = ?').get(id) as { summary: string } | undefined;
return { ...source, label: row?.summary ?? `Maintenance finding ${id.slice(0, 8)}`, href: row ? `/maintenance/${encodeURIComponent(id)}` : null };
}
const row = db().prepare('SELECT kind FROM maintenance_jobs WHERE id = ?').get(id) as { kind: string } | undefined;
return { ...source, label: row && row.kind !== 'legacy' ? `Maintenance ${row.kind} #${id}` : `Maintenance #${id}`,
href: row ? `/maintenance?view=activity#job-${encodeURIComponent(id)}` : null };
}
/** These are current pool snapshots, not a second admission/eligibility engine. */
function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): ProvisioningCapacity {
const settings = fleetSettings();
const pools: ProvisioningCapacity['pools'] = [];
if (deployment.action === 'create' || deployment.action === 'rebuild') {
const jobs = db().prepare(`SELECT id, server, state FROM server_deployments
WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health')
ORDER BY started_at, rowid`).all() as { id: string; server: string; state: string }[];
const holders = jobs.filter(job => job.state !== 'queued').map(({ id, server }) => ({ id, server }));
const order = jobs.filter(job => job.state === 'queued').findIndex(job => job.id === deployment.id);
pools.push({ kind: 'startup', limit: settings.createConcurrency, used: holders.length, holders,
requestOrder: order < 0 ? null : order + 1, ownsSlot: holders.some(holder => holder.id === deployment.id) });
}
if (['start', 'restart', 'rebuild'].includes(deployment.action)) {
const holders = db().prepare(`SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL
ORDER BY requested_at, server`).all() as { id: string | null; server: string }[];
pools.push({ kind: 'restart', limit: settings.restartConcurrency, used: holders.length, holders,
requestOrder: null, ownsSlot: holders.some(holder => holder.id === deployment.id) });
}
return { evaluatedAt, pools };
}
/** The default retains the full legacy event response. Page clients opt into a bounded window. */
export function getDeployment(id: string, options: { events?: 'all' | 'recent' } = {}): ProvisioningDetail | null {
return db().transaction(() => {
const row = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id) as
(ProvisioningDeployment & { observation_json: string | null; phase_key: string | null; phase_started_at: string | null; observed_at: string | null }) | undefined;
if (!row) return null;
const generatedAt = new Date().toISOString();
const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
const launch = recipeForDeployment(id);
const collection = db().prepare(`SELECT c.id, c.name FROM (
SELECT collection_id, deployment_id FROM server_collection_members
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
return { deployment, events: deploymentEvents(id, options.events === 'recent'),
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
warningMessages: warningMessages(deployment.warnings), generatedAt };
})();
}
// This stage is written only after Komodo has confirmed deployment deletion.
@@ -48,10 +127,13 @@ export function canCancelRemoval(id: string): boolean {
/** Competes atomically with the worker's claim, before any external removal. */
export function cancelRemoval(id: string): boolean {
return db().transaction(() => {
const finishedAt = new Date().toISOString();
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER);
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(finishedAt, id, REMOVING_SERVER);
if (!result.changes) return false;
recordDeploymentEvent(id, 'Queued removal cancelled. The server and its files were kept.');
recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome',
message: 'Queued removal cancelled. The server and its files were kept.',
details: { state: 'cancelled', finishedAt } } });
return true;
})();
}
@@ -76,8 +158,10 @@ export function canResumeCleanup(id: string): boolean {
export function resumeCleanup(id: string): boolean {
return db().transaction(() => {
if (!canResumeCleanup(id)) return false;
preserveOutcome(id);
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Resumed local cleanup after confirmed Komodo deletion');
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed Komodo deletion' } });
return true;
})();
}
@@ -100,29 +184,55 @@ export function canResumeRebuild(id: string): boolean {
}
export function resumeRebuild(id: string): boolean {
if (!canResumeRebuild(id)) return false;
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Resumed rebuild cleanup after confirmed Komodo deletion');
return true;
return db().transaction(() => {
if (!canResumeRebuild(id)) return false;
preserveOutcome(id);
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed Komodo deletion' } });
return true;
})();
}
/** Resume observation only. Never repeat a container creation or restart. */
export function resumeVerification(id: string): boolean {
return db().transaction(() => {
if (!canResumeVerification(id)) return false;
preserveOutcome(id);
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Resumed verification of the existing server');
deploymentStage(id, 'Waiting for Komodo health');
deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
event: { kind: 'resumed', message: 'Resumed verification of the existing server' } });
return true;
})();
}
export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
.run(state, new Date().toISOString(), result.message ?? null, JSON.stringify(result.warnings ?? []), id);
for (const warning of result.warnings ?? []) recordDeploymentEvent(id, warning);
recordDeploymentEvent(id, result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`);
db().transaction(() => {
const finishedAt = new Date().toISOString();
db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
.run(state, finishedAt, result.message ?? null, JSON.stringify(result.warnings ?? []), id);
for (const warning of result.warnings ?? []) recordProgress(id, { event: { kind: 'warning', message: warning } });
recordProgress(id, { blocker: null, event: { kind: 'outcome',
message: result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`,
details: { state, finishedAt, error: result.message ?? null, warnings: result.warnings ?? [] } } });
})();
}
/** Legacy failures also need a durable structured result before same-ID resume clears it. */
function preserveOutcome(id: string): void {
const row = db().prepare('SELECT state, finished_at, error, warnings, observation_json FROM server_deployments WHERE id = ?').get(id) as
{ state: string; finished_at: string | null; error: string | null; warnings: string; observation_json: string | null } | undefined;
if (!row || !['failed', 'interrupted', 'warning', 'done', 'cancelled'].includes(row.state)) return;
const previous = db().prepare("SELECT details_json FROM server_deployment_events WHERE deployment_id = ? AND kind = 'outcome' ORDER BY id DESC LIMIT 1")
.get(id) as { details_json: string | null } | undefined;
try {
const details = JSON.parse(previous?.details_json ?? 'null');
if (details?.state === row.state && details?.finishedAt === row.finished_at) return;
} catch { /* Preserve the row's known outcome when old event metadata is missing. */ }
recordProgress(id, { observedAt: readProgress(row)?.observedAt ?? null, event: { kind: 'outcome',
message: `${row.state === 'interrupted' ? 'Interrupted' : 'Recorded outcome'}: ${row.error ?? row.state}`,
details: { state: row.state, finishedAt: row.finished_at, error: row.error, warnings: warningMessages(row.warnings) } } });
}
export function listDeployments() {
@@ -132,9 +242,12 @@ export function listDeployments() {
}
export function interruptDeployments(): void {
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[];
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?,
error = 'The application restarted before this operation finished. Check the server before retrying.'
WHERE state = 'running'`).run(new Date().toISOString());
for (const { id } of running) recordDeploymentEvent(id, 'Interrupted by application restart. Check the server before retrying.');
db().transaction(() => {
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[];
const finishedAt = new Date().toISOString();
const error = 'The application restarted before this operation finished. Check the server before retrying.';
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running'`).run(finishedAt, error);
for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome',
message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } });
})();
}
+1
View File
@@ -4,4 +4,5 @@ export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWat
export type { LifecycleOperation, RestartSource } from './requests.js';
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextAvailablePort, nextLobbyNumber } from './instances.js';
export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';
export { deploymentActivity } from './activity.js';
export { restartSettling, restartRecoveryPending } from './restarts.js';
+182 -52
View File
@@ -1,5 +1,5 @@
import { fleetSettings } from '../settings.js';
import { probePlayerCount, probeServerStatus } from '../fleet/status-evidence.js';
import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js';
@@ -25,6 +25,8 @@ import {
import { MAP_CONFIG_FILE } from '../fleet/server-config.js';
import { recordScript } from '../fleet/inventory.js';
import { getVersion } from '../fleet/scripts.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress, type ProgressUpdate } from './progress.js';
export type CreateServerFailure =
| 'bad-request'
@@ -147,6 +149,13 @@ function buildSpec(args: {
}
let creating = false;
async function observeCommand<T>(operation: string, command: () => Promise<T>): Promise<T> {
try { return await command(); }
catch (err) {
recordProgress(operation, { command: { state: 'uncertain', message: 'No confirmed command result was received: ' + (err as Error).message } });
throw err;
}
}
interface PreparedCreation {
ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry;
}
@@ -225,7 +234,8 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
.run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
deploymentStage(id, 'Queued for server creation');
deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', recipeStatus: 'queued', observedAt: null,
blocker: { code: 'startup-queue', message: 'Waiting for startup admission and available capacity.', automatic: true } });
return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
})();
}
@@ -252,7 +262,8 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
deploymentStage(id, 'Waiting to rebuild the empty server');
deploymentStage(id, 'Waiting to rebuild the empty server', { phase: 'eligibility', recipeStatus: 'queued', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for safe rebuild eligibility checks.', automatic: true } });
return { ok: true as const, name, port: server.game_port!, deploymentId: id };
})();
}
@@ -266,12 +277,15 @@ export async function rebuildServer(id: string): Promise<void> {
if (blocker) {
// Keep the cleanup checkpoint intact when resuming after remote deletion.
if (job.stage === REMOVING_SERVER) finishDeployment(id, { ok: false, message: blocker });
else deploymentStage(id, blocker);
else deploymentStage(id, blocker, { phase: 'eligibility', blocker: betaRemovalEvidence(job.server, blocker), checks: [] });
return;
}
if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) {
// The cleanup checkpoint must survive capacity waits and app restarts.
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
const progress: ProgressUpdate = { phase: job.stage === REMOVING_SERVER ? 'cleanup' : 'eligibility',
blocker: { code: 'startup-capacity', message: 'Waiting for a server creation slot before rebuilding', automatic: true } };
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding', progress);
else recordProgress(id, progress);
return;
}
// Recheck prerequisites while the original instance still exists.
@@ -290,7 +304,9 @@ export async function rebuildServer(id: string): Promise<void> {
db().transaction(() => {
db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
deploymentStage(id, 'Queued for server creation');
deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', checks: [],
blocker: { code: 'startup-queue', message: 'Old instance removed; waiting for fresh-instance startup admission.', automatic: true },
checkpoint: { key: 'cleanup', label: 'Old instance files and registry removed' } });
})();
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
finally { destroying.delete(job.server); }
@@ -369,11 +385,15 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
// Resolve once more at provisioning so promotions made while queued also apply.
// Persist the exact attempted files for history, never as input to future launches.
const recipe = captureRecipe(game, mapEntry.slug);
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
.run(operation, JSON.stringify(recipe));
db().transaction(() => {
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
.run(operation, JSON.stringify(recipe));
deploymentStage(operation, 'Preparing server files', { phase: 'prepare', recipeStatus: 'attempted', blocker: null, checks: [],
checkpoint: { key: 'startup-queue', label: 'Admitted to startup' },
event: { kind: 'recipe', message: 'Promoted files selected for this creation attempt.', details: { capturedAt: recipe.capturedAt } } });
})();
const configFile = recipe.configFile;
deploymentStage(operation, 'Preparing server files');
const hostDir = serverDir(name);
// Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly
@@ -398,13 +418,18 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
}
// Step 2: Komodo CreateDeployment.
deploymentStage(operation, 'Creating Komodo deployment');
deploymentStage(operation, 'Creating Komodo deployment', { phase: 'create-deployment',
checkpoint: { key: 'prepare', label: 'Server files prepared' },
command: { state: 'attempted', message: 'Komodo deployment creation attempted.' } });
const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
const createRes = await komodoCreateDeployment(spec);
const createRes = await observeCommand(operation, () => komodoCreateDeployment(spec));
if (!createRes.ok) {
recordProgress(operation, { command: { state: createRes.kind === 'unreachable' || (createRes.httpStatus !== undefined && createRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: createRes.message } });
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` };
}
recordProgress(operation, { command: { state: 'accepted', message: 'Komodo created the deployment.' },
checkpoint: { key: 'remote-deployment-created', phase: 'create-deployment', label: 'Komodo deployment created' } });
// Step 3: DB row — insert *before* execute/Deploy so the sweep sees the
// server immediately even if Deploy hangs.
@@ -446,23 +471,31 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
void applyDeploymentTags(name, game, mapEntry.canonical);
// Step 5: execute/Deploy — actually start the container.
deploymentStage(operation, 'Starting container');
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
const deployRes = await komodoDeploy(name);
db().transaction(() => {
deploymentStage(operation, 'Starting container', { phase: 'container-action',
checkpoint: { key: 'create-deployment', label: 'Deployment created and server registered' },
command: { state: 'attempted', message: 'Container startup submission attempted.' } });
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
})();
const deployRes = await observeCommand(operation, () => komodoDeploy(name));
if (!deployRes.ok) {
if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
// A timeout or server error may arrive after the remote boot began.
// Preserve files, identity, and capacity until readiness is known.
recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message);
recordProgress(operation, { command: { state: 'uncertain', message: 'Startup may have begun: ' + deployRes.message } });
return { ok: true, name, port, deploymentId: operation };
}
// Container never started — undo everything so retrying is clean.
recordProgress(operation, { command: { state: 'rejected', message: 'Startup request rejected: ' + deployRes.message } });
await komodoDeleteDeployment(name).catch(() => {});
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` };
}
recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted startup; readiness still requires verification.' },
checkpoint: { key: 'container-action', label: 'Container startup accepted' } });
return { ok: true, name, port, deploymentId: operation };
}
@@ -515,7 +548,8 @@ export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerRe
const operation = db().transaction(() => {
const id = beginDeployment(name, 'destroy', triggeredBy);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty');
deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for safe removal eligibility checks.', automatic: true } });
return id;
})();
return { ok: true, warnings: [], deploymentId: operation };
@@ -528,6 +562,13 @@ export function processDestructions(): Promise<void> {
return destructionTick;
}
function betaRemovalEvidence(server: string, message: string): ProvisioningBlocker {
const beta = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') ORDER BY b.id LIMIT 1`)
.get(server) as { id: number } | undefined;
return { code: 'active-beta', message, automatic: true, href: beta ? `/betas/${beta.id}` : null, label: 'Blocking beta' };
}
async function runDestructions(): Promise<void> {
const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting'
AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at`).all(REMOVING_SERVER) as { id: string; server: string; stage: string; server_deleted_at: string | null }[];
@@ -553,37 +594,53 @@ async function runDestructions(): Promise<void> {
}
if (!getServer(operation.server)) {
if (hasActiveCreation(operation.server)) {
deploymentStage(operation.id, 'Waiting for server creation to finish');
const creatingOperation = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health') LIMIT 1`).get(operation.server) as { id: string } | undefined;
deploymentStage(operation.id, 'Waiting for server creation to finish', { phase: 'eligibility', checks: [],
blocker: { code: 'creation-pending', message: 'Waiting for server creation to finish', automatic: true,
href: creatingOperation ? `/provisioning/${encodeURIComponent(creatingOperation.id)}` : null, label: 'Creation operation' } });
continue;
}
finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
continue;
}
const blocker = activeBetaRemovalBlocker(operation.server);
if (blocker) { deploymentStage(operation.id, blocker); continue; }
if (blocker) { deploymentStage(operation.id, blocker, { phase: 'eligibility', checks: [], blocker: betaRemovalEvidence(operation.server, blocker) }); continue; }
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
let players = await probePlayerCount(operation.server);
const evidence = await probePlayerEvidence(operation.server);
let players = evidence.players;
let occupancy: NonNullable<ProgressUpdate['checks']>[number] = { key: 'occupancy', label: 'Player occupancy',
result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting', source: evidence.source,
checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players };
if (players === null) {
const health = await komodoInspectHealthy(operation.server);
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) {
players = 0;
occupancy = { key: 'stopped', label: 'Stopped container permits removal', result: 'pass', source: 'komodo', actual: health.state ?? null };
}
}
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
// Beta ownership may have changed while the network probes were in flight.
const latestBlocker = activeBetaRemovalBlocker(operation.server);
if (latestBlocker) { deploymentStage(operation.id, latestBlocker); continue; }
if (latestBlocker) { deploymentStage(operation.id, latestBlocker, { phase: 'eligibility', checks: [occupancy], blocker: betaRemovalEvidence(operation.server, latestBlocker) }); continue; }
if (players !== 0) {
deploymentStage(operation.id, players === null
const message = players === null
? 'Waiting for player count: server could not be reached'
: 'Waiting for server to empty: ' + players + ' player(s) online');
: 'Waiting for server to empty: ' + players + ' player(s) online';
deploymentStage(operation.id, message, { phase: 'eligibility', checks: [occupancy],
blocker: { code: players === null ? 'players-unknown' : 'players-present', message, automatic: true } });
continue;
}
if (!claimRemoval(operation.id)) continue;
deploymentStage(operation.id, 'Server is empty (0 players verified)');
deploymentStage(operation.id, 'Server is empty (0 players verified)', { phase: 'eligibility', checks: [occupancy], blocker: null,
checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' } });
const result = await destroyServerUnlocked(operation.server, operation.id);
finishDeployment(operation.id, result);
} catch (err) {
const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message);
if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message, {
phase: 'eligibility', checks: [], blocker: { code: 'players-unknown', message: (err as Error).message, automatic: true }
});
else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
} finally { destroying.delete(operation.server); }
}
@@ -601,11 +658,15 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
// 1: stop + remove the container. execute/DestroyDeployment queues the
// teardown; the response comes back before the container is actually gone.
deploymentStage(operation, 'Stopping and removing container');
const destroyRes = await komodoDestroyDeployment(name);
deploymentStage(operation, 'Stopping and removing container', { phase: 'remove-container', blocker: null, checks: [],
checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' },
command: { state: 'attempted', message: 'Container removal submission attempted.' } });
const destroyRes = await observeCommand(operation, () => komodoDestroyDeployment(name));
if (!destroyRes.ok) {
recordProgress(operation, { command: { state: destroyRes.kind === 'unreachable' || (destroyRes.httpStatus !== undefined && destroyRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: destroyRes.message } });
return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings };
}
recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted teardown; removal is not yet confirmed.' } });
// 2: wait for the container to actually disappear. Komodo refuses to
// DeleteDeployment while a container still exists for it, so poll the
@@ -619,12 +680,19 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
}
// 3: remove the deployment entity itself.
deploymentStage(operation, 'Deleting Komodo deployment');
const deleteRes = await komodoDeleteDeployment(name);
deploymentStage(operation, 'Deleting Komodo deployment', { phase: 'delete-deployment',
command: { state: 'attempted', message: 'Komodo deployment deletion attempted.' } });
const deleteRes = await observeCommand(operation, () => komodoDeleteDeployment(name));
if (!deleteRes.ok) {
recordProgress(operation, { command: { state: deleteRes.kind === 'unreachable' || (deleteRes.httpStatus !== undefined && deleteRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: deleteRes.message } });
return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
}
deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [],
command: { state: 'accepted', message: 'Komodo confirmed deployment deletion.' },
checkpoints: [{ key: 'remove-container', label: 'Container removal confirmed by deployment deletion' },
{ key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' }] });
return cleanupDestroyedServer(name, operation);
}
@@ -635,12 +703,14 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
const warnings: string[] = [];
// Persist the checkpoint before local work so failed cleanup can resume without RCON.
// Legacy operations already recorded this same stage after Komodo deletion.
deploymentStage(operation, REMOVING_SERVER);
deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [],
checkpoint: { key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' } });
// Host directory. Fall back to derived path if host_dir wasn't set on
// legacy rows.
const hostDir = row?.host_dir ?? serverDir(name);
try {
await rm(hostDir, { recursive: true, force: true });
recordProgress(operation, { checkpoint: { key: 'files-removed', label: 'Instance files removed', phase: 'cleanup' } });
} catch (err) {
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
}
@@ -648,7 +718,14 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
// The registry trigger preserves history and marks its old server identity.
// Live inventory, rollback snapshots, and restart slots still cascade away.
try {
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
db().transaction(() => {
const recovery = db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(name) as { operation_id: string | null } | undefined;
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
if (recovery?.operation_id) recordProgress(recovery.operation_id, { recovery: { state: 'released', operationId: recovery.operation_id,
message: 'Recovery capacity released when the old instance registry was removed.' } });
recordProgress(operation, { checkpoint: { key: 'registry-removed', label: 'Server registration removed', phase: 'cleanup' } });
if (!warnings.length) recordProgress(operation, { checkpoint: { key: 'cleanup', label: 'Local files and registry cleanup completed' } });
})();
} catch (err) {
return {
ok: false,
@@ -684,8 +761,11 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
{ ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
if (!result.ok) finishDeployment(id, result);
else {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for Komodo health');
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', checks: [],
blocker: { code: 'container-readiness', message: 'Waiting for container readiness.', automatic: true } });
})();
}
return result;
} catch (err) {
@@ -693,9 +773,13 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
if (server?.game_port === job.port) {
// A lost response may have happened after Komodo started the server.
// Keep its slot and observe readiness instead of starting another boot.
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
deploymentStage(id, 'Verifying existing server after interrupted startup');
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
deploymentStage(id, 'Verifying existing server after interrupted startup', { phase: 'verify-container', checks: [],
command: { state: 'uncertain', message: 'Startup response interrupted: ' + (err as Error).message },
blocker: { code: 'container-readiness', message: 'Observing the existing instance after an uncertain startup.', automatic: true } });
})();
} else finishDeployment(id, { ok: false, message: (err as Error).message });
throw err;
} finally { creating = false; }
@@ -722,19 +806,32 @@ export function creationQueue() {
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
}
function creationBlocker(id: string): string | null {
const dependency = db().prepare(`SELECT d.server, d.state FROM server_creation_dependencies dep
function creationBlocker(id: string): ProvisioningBlocker | null {
const dependency = db().prepare(`SELECT d.id, d.server, d.state FROM server_creation_dependencies dep
JOIN server_deployments d ON d.id = dep.removal_id
WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { server: string; state: string } | undefined;
if (dependency) return `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`;
WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { id: string; server: string; state: string } | undefined;
if (dependency) return { code: 'removal-dependency', message: `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`,
automatic: true, href: `/provisioning/${encodeURIComponent(dependency.id)}`, label: 'Required removal' };
const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port
WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined;
return occupied ? `Waiting for port to be released by ${occupied.name}` : null;
return occupied ? { code: 'port-occupied', message: `Waiting for port to be released by ${occupied.name}`, automatic: true,
href: `/servers/${encodeURIComponent(occupied.name)}`, label: 'Server using this port' } : null;
}
function stillCheckingCreation(id: string): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments d JOIN server_creation_jobs j ON j.deployment_id = d.id
JOIN servers s ON s.name = j.server AND s.game_port = j.port
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state = 'awaiting-health'`).get(id);
}
async function runCreations(): Promise<CreationOutcomes> {
const outcomes: CreationOutcomes = new Map();
if (creating || destroying.size > 0) return outcomes;
if (creating || destroying.size > 0) {
for (const job of creationQueue().queued) recordProgress(job.id, { phase: 'startup-queue', blocker: {
code: 'startup-worker-busy', message: 'Waiting for in-flight instance creation or removal to finish.', automatic: true
} });
return outcomes;
}
// Restart recovery is observation-only: never repeat external creation or
// deployment. A known interrupted startup must retain its readiness gate.
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
@@ -752,7 +849,8 @@ async function runCreations(): Promise<CreationOutcomes> {
AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id);
if (removing) {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for requested server removal after interrupted startup');
deploymentStage(id, 'Waiting for requested server removal after interrupted startup', { checks: [],
blocker: { code: 'removal-pending', message: 'Waiting for the requested removal; the interrupted startup still occupies capacity.', automatic: true } });
}
}
@@ -766,21 +864,51 @@ async function runCreations(): Promise<CreationOutcomes> {
finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
continue;
}
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
const removal = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server) as { id: string } | undefined;
if (removal) {
recordProgress(id, { checks: [], blocker: { code: 'removal-pending', message: 'Creation verification is held while server removal is pending.', automatic: true,
href: `/provisioning/${encodeURIComponent(removal.id)}`, label: 'Removal operation' } });
continue;
}
const checks: NonNullable<ProgressUpdate['checks']> = [];
try {
const komodo = await komodoInspectHealthy(job.server);
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
if (!stillCheckingCreation(id)) continue;
checks.push({ key: 'container', label: 'Container readiness', source: 'komodo', result: komodo.ok ? 'pass' : 'waiting',
checkedAt: new Date().toISOString(), actual: komodo.state ?? null, message: komodo.ok ? null : komodo.reason });
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason, { phase: 'verify-container', checks,
blocker: { code: 'container-readiness', message: komodo.reason, automatic: true } }); continue; }
const status = await probeServerStatus(job.server, job.submitted_at);
if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason); continue; }
if (!stillCheckingCreation(id)) continue;
checks.push({ key: 'game', label: 'Game status', source: status.ok ? status.source : null, result: status.ok ? 'pass' : 'waiting',
checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null, message: status.ok ? null : status.reason });
if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason, { phase: 'verify-game', checks,
blocker: { code: 'game-readiness', message: status.reason, automatic: true } }); continue; }
const actualMap = status.map;
checks.push({ key: 'map', label: 'Selected engine map', source: status.source, result: actualMap === job.expected_map ? 'pass' : 'waiting',
checkedAt: status.observedAt, receivedAt: status.receivedAt, expected: job.expected_map, actual: actualMap });
if (actualMap !== job.expected_map) {
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map'));
const message = 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map');
deploymentStage(id, message, { phase: 'verify-map', checks, blocker: { code: 'map-mismatch', message, automatic: true,
href: `/servers/${encodeURIComponent(job.server)}?tab=config`, label: 'Map configuration' } });
continue;
}
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap);
finishDeployment(id, { ok: true });
db().transaction(() => {
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap, {
phase: 'verify-map', checks, blocker: null, checkpoints: [
{ key: 'container-action', label: 'Container startup verified' },
{ key: 'verify-container', label: 'Container readiness verified' },
{ key: 'verify-game', label: 'Game status verified' },
{ key: 'verify-map', label: 'Selected engine map verified' }
]
});
finishDeployment(id, { ok: true });
})();
} catch (err) {
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
if (!stillCheckingCreation(id)) continue;
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', source: null, result: 'unknown', message: (err as Error).message });
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message, { phase: checks.length > 1 ? 'verify-game' : 'verify-container', checks,
blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } });
}
}
@@ -790,7 +918,9 @@ async function runCreations(): Promise<CreationOutcomes> {
const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length);
const eligible = queue.queued.filter(({ id }) => {
const blocker = creationBlocker(id);
if (blocker) deploymentStage(id, blocker);
if (blocker) deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker });
else recordProgress(id, { phase: 'startup-queue', blocker: { code: available ? 'startup-queue' : 'startup-capacity',
message: available ? 'Eligible for startup admission in request order.' : 'Waiting for startup capacity to be released.', automatic: true } });
return !blocker;
});
for (const { id } of eligible.slice(0, available)) {
@@ -801,7 +931,7 @@ async function runCreations(): Promise<CreationOutcomes> {
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
const blocker = creationBlocker(id);
if (blocker) {
deploymentStage(id, blocker);
deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker });
continue;
}
try { outcomes.set(id, await provisionCreation(id)); }
+120
View File
@@ -0,0 +1,120 @@
import { db } from '../db/index.js';
import { provisioningPhases, type ProvisioningPhase, type ProvisioningProgress, type ProvisioningCheck,
type ProvisioningCheckpoint, type ProvisioningBlocker, type RecoveryObservation } from '../../provisioning-detail.js';
export type CheckInput = Omit<ProvisioningCheck, 'checkedAt' | 'receivedAt' | 'expected' | 'actual' | 'message'> &
Partial<Pick<ProvisioningCheck, 'checkedAt' | 'receivedAt' | 'expected' | 'actual' | 'message'>>;
export type CheckpointInput = Omit<ProvisioningCheckpoint, 'at' | 'phase'> & { at?: string; phase?: ProvisioningPhase | null };
export interface ProgressUpdate {
phase?: ProvisioningPhase | null;
blocker?: ProvisioningBlocker | null;
/** A complete evaluation; omitted checks are no longer current evidence. */
checks?: CheckInput[];
checkpoint?: CheckpointInput;
checkpoints?: CheckpointInput[];
command?: (Omit<NonNullable<ProvisioningProgress['command']>, 'at'> & { at?: string }) | null;
recovery?: (Omit<RecoveryObservation, 'at'> & { at?: string }) | null;
recipeStatus?: ProvisioningProgress['recipeStatus'];
observedAt?: string | null;
event?: { kind: string; message: string; details?: Record<string, unknown> };
}
const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === 'object' && !Array.isArray(value);
const nullableString = (value: unknown) => value === null || typeof value === 'string';
const phase = (value: unknown): value is ProvisioningPhase => provisioningPhases.includes(value as ProvisioningPhase);
const scalar = (value: unknown) => nullableString(value) || typeof value === 'number' && Number.isFinite(value);
/** Invalid/old snapshots never manufacture evidence. Raw legacy stages remain available. */
export function readProgress(row: { observation_json?: string | null }): ProvisioningProgress | null {
if (!row.observation_json) return null;
try {
const v: unknown = JSON.parse(row.observation_json);
if (!object(v) || v.version !== 1 || !(v.phase === null || phase(v.phase)) || !nullableString(v.phaseStartedAt)
|| !nullableString(v.observedAt) || !Array.isArray(v.checks) || !Array.isArray(v.checkpoints)) return null;
if (v.blocker !== null && (!object(v.blocker) || typeof v.blocker.code !== 'string' || typeof v.blocker.message !== 'string'
|| typeof v.blocker.automatic !== 'boolean' || (v.blocker.href !== undefined && !nullableString(v.blocker.href))
|| (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null;
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'occupancy', 'stopped'].includes(String(c.key))
&& typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result))
&& (c.source === null || ['komodo', 'rcon', 'telemetry', 'combined'].includes(String(c.source)))
&& typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null;
if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string'
&& typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null;
if (v.command !== null && (!object(v.command) || !['attempted', 'accepted', 'rejected', 'uncertain'].includes(String(v.command.state))
|| typeof v.command.at !== 'string' || typeof v.command.message !== 'string')) return null;
if (v.recovery !== null && (!object(v.recovery) || !['held', 'released', 'transferred', 'unknown'].includes(String(v.recovery.state))
|| typeof v.recovery.at !== 'string' || typeof v.recovery.message !== 'string'
|| (v.recovery.operationId !== undefined && !nullableString(v.recovery.operationId)))) return null;
if (v.recipeStatus !== null && v.recipeStatus !== 'queued' && v.recipeStatus !== 'attempted') return null;
return v as unknown as ProvisioningProgress;
} catch { return null; }
}
/**
* Records facts at the existing execution boundary, never performs a probe or command.
* Can be nested inside the caller's transaction with its state/legacy-stage writes.
*/
export function recordProgress(id: string, update: ProgressUpdate): void {
const handle = db();
handle.transaction(() => {
const row = handle.prepare('SELECT observation_json FROM server_deployments WHERE id = ?').get(id) as
{ observation_json: string | null } | undefined;
if (!row) return;
const now = new Date().toISOString();
const previous = readProgress(row);
const next: ProvisioningProgress = previous ? { ...previous, checkpoints: [...previous.checkpoints] } : {
version: 1, phase: null, phaseStartedAt: null, observedAt: null, blocker: null, checks: [], checkpoints: [],
command: null, recovery: null, recipeStatus: null
};
const events: { kind: string; message: string; details?: Record<string, unknown> }[] = [];
if (update.phase !== undefined && update.phase !== next.phase) {
next.phase = update.phase;
next.phaseStartedAt = update.phase === null ? null : now;
next.blocker = null;
}
next.observedAt = update.observedAt === undefined ? now : update.observedAt;
if (update.blocker !== undefined) {
const old = next.blocker;
next.blocker = update.blocker;
// Counts/error wording can change on every poll without a new kind of hold.
if (update.blocker && (old?.code !== update.blocker.code || old?.href !== update.blocker.href)) {
events.push({ kind: 'blocker', message: update.blocker.message,
details: { code: update.blocker.code, href: update.blocker.href ?? null } });
}
}
if (update.checks !== undefined) {
next.checks = update.checks.map(check => ({ ...check, checkedAt: check.checkedAt ?? now,
receivedAt: check.receivedAt ?? null, expected: check.expected ?? null, actual: check.actual ?? null,
message: check.message ?? null }));
}
for (const input of [...(update.checkpoints ?? []), ...(update.checkpoint ? [update.checkpoint] : [])]) {
if (next.checkpoints.some(c => c.key === input.key)) continue;
const point: ProvisioningCheckpoint = { ...input, at: input.at ?? now, phase: input.phase ?? (phase(input.key) ? input.key : next.phase) };
next.checkpoints.push(point);
events.push({ kind: 'checkpoint', message: point.label, details: { key: point.key, at: point.at } });
}
if (update.command !== undefined) {
if (!update.command) next.command = null;
else if (next.command?.state !== update.command.state || next.command?.message !== update.command.message) {
next.command = { ...update.command, at: update.command.at ?? now };
events.push({ kind: 'command', message: next.command.message, details: { state: next.command.state, at: next.command.at } });
}
}
if (update.recovery !== undefined) {
if (!update.recovery) next.recovery = null;
else {
const changed = next.recovery?.state !== update.recovery.state || next.recovery?.operationId !== update.recovery.operationId;
next.recovery = { ...update.recovery, at: changed ? update.recovery.at ?? now : next.recovery!.at };
if (changed) events.push({ kind: 'recovery', message: next.recovery.message,
details: { state: next.recovery.state, at: next.recovery.at, operationId: next.recovery.operationId ?? null } });
}
}
if (update.recipeStatus !== undefined) next.recipeStatus = update.recipeStatus;
if (update.event) events.push(update.event);
handle.prepare(`UPDATE server_deployments SET phase_key = ?, phase_started_at = ?, observed_at = ?, observation_json = ? WHERE id = ?`)
.run(next.phase, next.phaseStartedAt, next.observedAt, JSON.stringify(next), id);
const insert = handle.prepare(`INSERT INTO server_deployment_events (deployment_id, created_at, message, kind, phase_key, details_json)
VALUES (?, ?, ?, ?, ?, ?)`);
for (const event of events) insert.run(id, now, event.message, event.kind, next.phase, event.details ? JSON.stringify(event.details) : null);
})();
}
+43 -12
View File
@@ -6,6 +6,7 @@ import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEve
import { queueRebuildServer } from './instances.js';
import { restartRequestInFlight } from './restarts.js';
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
import { recordProgress } from './progress.js';
export interface LifecycleOperation {
id: string; server: string; action: string; state: string; stage: string;
@@ -67,7 +68,8 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
if (!id) {
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty');
deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at);
}
db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)')
@@ -79,12 +81,17 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
/** Upgrade observation only: existing restart intent is never resubmitted. */
function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void {
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
{ operation_id: string | null; requested_at: string; healthy_at: string | null; players: number | null } | undefined;
const slot = db().prepare('SELECT operation_id, requested_at, checked_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
{ operation_id: string | null; requested_at: string; checked_at: string | null; healthy_at: string | null; players: number | null } | undefined;
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id);
recordProgress(id, { command: { state: 'attempted', at: submittedAt, message: 'Adopted a previously recorded submission; no command is replayed.' } });
if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) {
const finishedAt = new Date().toISOString();
const error = 'Existing restart cannot be correlated safely. Inspect the server before retrying.';
db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?")
.run(new Date().toISOString(), 'Existing restart cannot be correlated safely. Inspect the server before retrying.', id);
.run(finishedAt, error, id);
recordProgress(id, { recovery: { state: 'unknown', message: 'Existing restart recovery could not be correlated with this operation.', operationId: id },
observedAt: null, event: { kind: 'outcome', message: error, details: { state: 'interrupted', finishedAt, error } } });
return;
}
if (slot && slot.requested_at >= submittedAt) {
@@ -96,16 +103,24 @@ function adoptSubmittedRestart(id: string, server: string, submittedAt: string):
}
if (slot?.healthy_at && slot.requested_at >= submittedAt) {
db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id);
recordProgress(id, { observedAt: slot.healthy_at, event: { kind: 'outcome', message: 'Adopted recorded recovery completion.',
details: { state: 'done', finishedAt: slot.healthy_at, error: null, warnings: [] } } });
}
deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again');
deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again', {
phase: 'verify-container', checks: [], blocker: null, observedAt: slot?.checked_at ?? null,
recovery: { state: slot?.healthy_at && slot.requested_at >= submittedAt ? 'released' : 'held',
at: slot?.healthy_at && slot.requested_at >= submittedAt ? slot.healthy_at : undefined,
message: slot?.healthy_at && slot.requested_at >= submittedAt ? 'Adopted recorded recovery completion.' : 'Observing the previously submitted restart without replaying it.', operationId: id }
});
}
export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean {
return db().transaction(() => {
const finishedAt = new Date().toISOString();
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL
WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL`)
.run(new Date().toISOString(), reason, id);
if (result.changes) recordDeploymentEvent(id, reason);
.run(finishedAt, reason, id);
if (result.changes) recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt, error: null, warnings: [] } } });
return result.changes > 0;
})();
}
@@ -143,7 +158,8 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
const id = beginDeployment(server, action);
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
blocker: { code: 'eligibility', message: action === 'start' ? 'Waiting for start eligibility and recovery capacity.' : 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
return { ok: true as const, deploymentId: id };
})();
}
@@ -177,7 +193,8 @@ function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', inci
} else {
id = beginDeployment(server, action, `watchdog:${incidentId}`);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty');
deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
}
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server);
return { ok: true, deploymentId: id };
@@ -209,8 +226,14 @@ export function requestWatchdogLifecycle(server: string, action: 'restart' | 're
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
if (retained && !restartRequestInFlight(server)) {
db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(next.deploymentId, server, retained.operation_id);
if (transferred.changes) {
recordProgress(retained.operation_id, { recovery: { state: 'transferred', operationId: next.deploymentId,
message: 'Recovery capacity transferred to the explicit new incident attempt.' } });
recordProgress(next.deploymentId, { recovery: { state: 'held', operationId: next.deploymentId,
message: 'Recovery capacity retained from the prior incident attempt.' } });
}
recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId);
recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id);
}
@@ -249,8 +272,16 @@ export function escalateWatchdogRestart(incidentId: string, operationId: string,
const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId;
if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason });
else recordDeploymentEvent(operationId, reason);
if (slot) db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(replacement.deploymentId, reason, operation.server, operationId);
if (slot) {
const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(replacement.deploymentId, reason, operation.server, operationId);
if (transferred.changes) {
recordProgress(operationId, { recovery: { state: 'transferred', operationId: replacement.deploymentId,
message: 'Recovery capacity transferred to the rebuild operation.' } });
recordProgress(replacement.deploymentId, { recovery: { state: 'held', operationId: replacement.deploymentId,
message: 'Recovery capacity retained from the previous restart.' } });
}
}
recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server');
return replacement;
})();
+78 -30
View File
@@ -5,6 +5,8 @@ import { probeServerStatus } from '../fleet/status-evidence.js';
import { deploymentStage, finishDeployment } from './history.js';
import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
import type { RuntimeAction } from '../../provisioning.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress, type ProgressUpdate } from './progress.js';
// Komodo queues restart work. Don't mistake the still-running old container
// for a completed restart when an API kick immediately follows the request.
@@ -38,21 +40,28 @@ export function restartSettling(server: string): boolean {
}
export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
return rolloutBlocker(server, operationId, action)?.message ?? null;
}
export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
return 'Waiting for the current provisioning operation';
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
if (conflict) {
return { code: 'operation-conflict', message: 'Waiting for the current provisioning operation', automatic: true,
href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' };
}
const slot = slotFor(server);
const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS (
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId);
if (slot?.healthy_at === null && !ownHandoff) {
return `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`;
return { code: 'previous-recovery', message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`, automatic: true,
href: slot.operation_id ? `/provisioning/${encodeURIComponent(slot.operation_id)}` : null, label: 'Previous operation' };
}
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency
? `Queued to ${action}; waiting for another server to recover` : null;
? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null;
}
export type RolloutRestartResult =
@@ -71,6 +80,7 @@ export async function restartInRollout(server: string, options: { operationId: s
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
.run(server, new Date().toISOString(), operationId);
recordProgress(operationId, { blocker: null, recovery: { state: 'held', message: 'Restart recovery capacity reserved.', operationId } });
return null;
}).immediate();
if (waitReason) return { kind: 'queued', message: waitReason };
@@ -81,12 +91,20 @@ export async function restartInRollout(server: string, options: { operationId: s
applyStagedFilesForOperation(operationId, server);
} catch (err) {
const message = `File activation failed: ${(err as Error).message}`;
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
finishDeployment(operationId, { ok: false, message });
db().transaction(() => {
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
recordProgress(operationId, { recovery: { state: 'released', message: 'Capacity released before any remote command: file activation failed.', operationId } });
finishDeployment(operationId, { ok: false, message });
})();
return { kind: 'failed', message };
}
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
db().transaction(() => {
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container', {
phase: 'container-action', blocker: null, checks: [], checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' },
command: { state: 'attempted', message: `${options.action === 'start' ? 'Start' : 'Restart'} submission attempted.` }
});
})();
requestsInFlight.set(server, operationId);
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
@@ -94,24 +112,38 @@ export async function restartInRollout(server: string, options: { operationId: s
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
if (result.ok) {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)');
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)', {
phase: 'verify-container', command: { state: 'accepted', message: 'Komodo accepted the request; recovery still requires verification.' },
checkpoint: { key: 'container-action', label: 'Container command accepted' },
blocker: { code: 'recovery-settling', message: 'Waiting for the recovery settling interval before checking health.', automatic: true }
});
})();
return { kind: 'started' };
}
// These failures cannot have queued a remote restart. Other failures
// (especially a timeout) may have: retain capacity until recovery.
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
} else {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
}
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
db().transaction(() => {
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
recordProgress(operationId, { command: { state: 'rejected', message: 'The command could not be submitted: ' + result.message },
recovery: { state: 'released', message: 'Capacity released because no remote command could be submitted.', operationId } });
} else {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + result.message } });
}
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
})();
return { kind: 'failed', message: `komodo: ${result.message}` };
} catch (err) {
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
const message = `komodo: ${(err as Error).message}`;
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
finishDeployment(operationId!, { ok: false, message });
db().transaction(() => {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + (err as Error).message } });
finishDeployment(operationId!, { ok: false, message });
})();
return { kind: 'failed', message };
} finally {
// Start the grace period at completion, including slow/failed requests.
@@ -145,31 +177,47 @@ async function refresh(): Promise<void> {
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
let error: string | null = null;
let players: number | null = null;
const checks: NonNullable<ProgressUpdate['checks']> = [];
try {
const health = await komodoInspectHealthy(slot.server);
checks.push({ key: 'container', label: 'Container readiness', result: health.ok ? 'pass' : 'waiting', source: 'komodo', actual: health.state ?? null,
checkedAt: new Date().toISOString(), message: health.ok ? null : health.reason });
if (!health.ok) {
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
} else {
const status = await probeServerStatus(slot.server, slot.requested_at);
checks.push({ key: 'game', label: 'Game status', result: status.ok ? 'pass' : 'waiting', source: status.ok ? status.source : null,
checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null,
message: status.ok ? null : status.reason });
if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`;
else players = status.players;
}
} catch (err) {
error = `health check: ${(err as Error).message}`;
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', result: 'unknown', source: null, message: error });
}
const now = new Date().toISOString();
const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
if (!checked.changes) continue;
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
finishDeployment(slot.operation_id, { ok: true });
db().transaction(() => {
const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
if (!checked.changes) return;
if (slot.operation_id) recordProgress(slot.operation_id, {
phase: checks.some(check => check.key === 'game') ? 'verify-game' : 'verify-container', checks, observedAt: now,
blocker: error ? { code: 'recovery-health', message: error, automatic: true } : null,
recovery: { state: error ? 'held' : 'released', at: now, operationId: slot.operation_id,
message: error ? 'Restart capacity remains reserved while recovery is checked.' : 'Container and game recovery verified; restart capacity released.' },
checkpoints: error ? [] : [{ key: 'verify-container', label: 'Container readiness verified', at: now }, { key: 'verify-game', label: 'Game status verified', at: now }]
});
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
finishDeployment(slot.operation_id, { ok: true });
}
}
}
})();
}
}
+83 -32
View File
@@ -5,10 +5,12 @@ import { getServer } from '../fleet/servers.js';
import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
import { processCreations, processDestructions, rebuildServer } from './instances.js';
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
import { probePlayerCount } from '../fleet/status-evidence.js';
import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from './restarts.js';
import { probePlayerEvidence } from '../fleet/status-evidence.js';
import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
import { cancelProvisioning } from './requests.js';
import type { RuntimeAction } from '../../provisioning.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress } from './progress.js';
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
@@ -18,7 +20,7 @@ const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM pro
const priority = (sources: Source[]) => sources.some(s => s.kind === 'watchdog') ? 2 : sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2;
/** Source work may hold or withdraw a request, but never executes it. */
function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' {
function sourceEligibility(operation: Operation, onHold?: (source: Source) => void): 'ready' | 'held' | 'cancelled' {
const sources = sourcesFor(operation.id);
if (!sources.length) return 'ready';
// Watchdog intent gates every linked source, including its drift campaign.
@@ -26,7 +28,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(source.source_id, operation.server)
|| getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
const incident = db().prepare('SELECT kind FROM watchdog_incidents WHERE id = ?').get(source.source_id) as { kind: 'health' | 'drift' };
if (!maintenanceAllowed(incident.kind)) return 'held';
if (!maintenanceAllowed(incident.kind)) { onHold?.(source); return 'held'; }
}
let held = false;
let ready = false;
@@ -36,7 +38,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
.get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined;
if (row?.provisioning_id !== operation.id || !row.files_changed) continue;
if (['pending','failed'].includes(row.restart_state)) ready = true;
if (['held','manual'].includes(row.restart_state)) held = true;
if (['held','manual'].includes(row.restart_state)) { held = true; onHold?.(source); }
} else if (source.kind === 'beta') {
const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`)
@@ -44,12 +46,12 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue;
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
if (!row.outcome || ['active','passed'].includes(row.outcome)) ready = true;
if (row.outcome === 'paused') held = true;
if (row.outcome === 'paused') { held = true; onHold?.(source); }
} else if (source.kind === 'maintenance') {
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) {
if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) return 'held';
if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) { onHold?.(source); return 'held'; }
ready = true;
}
} else ready = true;
@@ -57,26 +59,31 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
return held ? 'held' : ready ? 'ready' : 'cancelled';
}
function blocker(server: string, id: string, action: RuntimeAction): string | null {
function blocker(server: string, id: string, action: RuntimeAction): ProvisioningBlocker | null {
const sources = sourcesFor(id);
const rank = priority(sources);
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[];
const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number };
if (others.some(other => other.state !== 'waiting' || other.action === 'destroy'
|| priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid))) {
return 'Waiting for another provisioning operation on this server';
const conflict = others.find(other => other.state !== 'waiting' || other.action === 'destroy'
|| priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid));
if (conflict) {
return { code: 'operation-conflict', message: 'Waiting for another provisioning operation on this server', automatic: true,
href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' };
}
// Beta activation must not wait on the campaign rows that the beta itself
// holds. Campaign/maintenance callers likewise never block on their own job.
if (rank > 0 && db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
const beta = rank > 0 ? db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server) as { id: string } | undefined : undefined;
if (beta) return { code: 'beta-work', message: 'Waiting for beta work on this server', automatic: true, href: `/betas/${encodeURIComponent(beta.id)}`, label: 'Blocking beta' };
const campaign = rank > 1 ? db().prepare(`SELECT ds.deployment_id FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
LIMIT 1`).get(server, id)) return 'Waiting for campaign work on this server';
return rolloutWaitReason(server, id, action);
LIMIT 1`).get(server, id) as { deployment_id: string } | undefined : undefined;
if (campaign) return { code: 'campaign-work', message: 'Waiting for campaign work on this server', automatic: true,
href: `/campaigns/${encodeURIComponent(campaign.deployment_id)}`, label: 'Blocking campaign' };
return rolloutBlocker(server, id, action);
}
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
@@ -100,7 +107,9 @@ async function run(): Promise<void> {
try { await processOperation(operation); }
catch (err) {
// Observation failures are retried; remote mutations are never replayed.
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message);
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, {
checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true }
});
}
}
}
@@ -112,36 +121,65 @@ async function processOperation(operation: Operation): Promise<void> {
// Watchdog is paused; never strand the reserved replacement identity.
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
if (operation.state === 'waiting') {
const eligibility = sourceEligibility(operation);
const heldBy: Source[] = [];
const eligibility = sourceEligibility(operation, source => heldBy.push(source));
if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
if (eligibility === 'held') { deploymentStage(id, 'Waiting for the requester to resume'); return; }
if (eligibility === 'held') {
const source = heldBy[0];
const href = source ? source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}`
: source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}`
: source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}`
: `/maintenance?view=activity#job-${encodeURIComponent(source.source_id)}` : null;
deploymentStage(id, 'Waiting for the requester to resume', { phase: 'eligibility', checks: [],
blocker: { code: 'requester-held', message: 'Waiting for the requester to resume', automatic: false, href, label: 'Requesting work' } });
return;
}
}
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog');
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration', {
phase: 'eligibility', checks: [], blocker: { code: 'configuration', message: 'Waiting for Komodo and RCON configuration', automatic: false, href: '/settings', label: 'Integration settings' }
}); return; }
if (operation.state === 'awaiting-health') {
const health = await komodoInspectHealthy(server);
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(id)) return;
recordProgress(id, { phase: 'verify-stopped', checks: [{ key: 'stopped', label: 'Container stopped', source: 'komodo',
result: isStopped(health.state) ? 'pass' : health.state ? 'waiting' : 'unknown', actual: health.state ?? null,
message: health.ok ? null : health.reason }], blocker: isStopped(health.state) ? null
: { code: 'stopped-state', message: 'Waiting for an explicit stopped container state.', automatic: true } });
if (isStopped(health.state)) {
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
deploymentStage(id, 'Verified container stopped');
finishDeployment(id, { ok: true });
db().transaction(() => {
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
deploymentStage(id, 'Verified container stopped', { checkpoint: { key: 'verify-stopped', label: 'Stopped container state verified' } });
finishDeployment(id, { ok: true });
})();
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
return;
}
const reason = blocker(server, id, action);
if (reason) { deploymentStage(id, reason); return; }
if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
const health = await komodoInspectHealthy(server);
if (action !== 'start' && !isStopped(health.state)) {
const players = await probePlayerCount(server);
const evidence = await probePlayerEvidence(server);
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
const players = evidence.players;
recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',
message: players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`, automatic: true } });
db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
if (players !== 0) {
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
return;
}
} else if (action !== 'start') {
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
recordProgress(id, { phase: 'eligibility', blocker: null, checks: [{ key: 'stopped', label: 'Stopped container permits disruption',
source: 'komodo', result: 'pass', actual: health.state ?? null }] });
}
// Requests can arrive during the probes. Recheck before claiming the action.
const latestReason = blocker(server, id, action);
if (latestReason) { deploymentStage(id, latestReason); return; }
if (latestReason) { deploymentStage(id, latestReason.message, { phase: 'eligibility', blocker: latestReason }); return; }
if (sourceEligibility(operation) !== 'ready') return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
if (action === 'rebuild') { await rebuildServer(id); return; }
@@ -155,13 +193,26 @@ async function processOperation(operation: Operation): Promise<void> {
}
return;
}
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
deploymentStage(id, 'Stopping container');
db().transaction(() => {
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
deploymentStage(id, 'Stopping container', { phase: 'container-action', blocker: null, checks: [],
checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, command: { state: 'attempted', message: 'Stop submission attempted.' } });
})();
// Persist intent first; a lost reply must not issue another stop later.
try {
const result = await komodoRuntimeAction(server, 'stop');
if (!result.ok) { finishDeployment(id, { ok: false, message: result.message }); return; }
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for container to stop');
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
if (!result.ok) {
recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } });
finishDeployment(id, { ok: false, message: result.message }); return;
}
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for container to stop', { phase: 'verify-stopped',
checkpoint: { key: 'container-action', label: 'Stop command accepted' },
command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } });
})();
} catch (err) {
recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } });
finishDeployment(id, { ok: false, message: (err as Error).message });
}
}
+3 -3
View File
@@ -1,11 +1,11 @@
<script lang="ts">
import { timeFormat } from '$ui/stores/time-zone.js';
import type { LaunchRecipeSummary } from '$lib/deployment-plan.js';
let { recipe, historical = false }: { recipe: LaunchRecipeSummary; historical?: boolean } = $props();
let { recipe, historical = false, selectionStatus = null }: { recipe: LaunchRecipeSummary; historical?: boolean; selectionStatus?: 'queued' | 'attempted' | null } = $props();
</script>
<div class="recipe">
<div class="recipe-heading"><div><span class="eyebrow">{historical ? 'Recorded launch files' : 'Current promoted setup'}</span><h3>{recipe.mapName}</h3></div><span class="game">{recipe.game === 't6' ? 'Black Ops II' : 'Black Ops I'}</span></div>
<div class="recipe-heading"><div><span class="eyebrow">{historical ? selectionStatus === 'queued' ? 'Queued setup capture' : selectionStatus === 'attempted' ? 'Selected for this attempt' : 'Recorded launch files' : 'Current promoted setup'}</span><h3>{recipe.mapName}</h3></div><span class="game">{recipe.game === 't6' ? 'Black Ops II' : 'Black Ops I'}</span></div>
<dl>
<div><dt>Configuration</dt><dd>{recipe.configSource}</dd></div>
<div><dt>Container image</dt><dd class="mono">{recipe.image}</dd></div>
@@ -14,7 +14,7 @@
<details>
<summary>{recipe.files.length} managed file{recipe.files.length === 1 ? '' : 's'} {historical ? 'recorded for this deployment' : 'in the current setup'}</summary>
<ul>{#each recipe.files as file}<li><div><strong>{file.name}</strong><span>{file.version !== null ? `v${file.version}` : 'Generated'} · {file.kind} · {file.size.toLocaleString()} bytes</span></div><code>{file.path}</code><small title={file.sha256}>SHA-256 {file.sha256.slice(0, 12)}</small></li>{/each}</ul>
<p>{historical ? 'Recorded' : 'Previewed'} {$timeFormat.dateTime(recipe.capturedAt)}. {historical ? 'This records the files selected for this deployment. Launch again uses the latest promoted setup for this map.' : 'This preview shows the promoted setup at the time above. The deployment history records the files selected when provisioning begins.'}</p>
<p>{historical ? 'Recorded' : 'Previewed'} {$timeFormat.dateTime(recipe.capturedAt)}. {historical ? selectionStatus === 'queued' ? 'Versions are selected again when creation begins. This queued capture does not pin them.' : selectionStatus === 'attempted' ? 'These files were selected for this attempt; this does not confirm installation succeeded. Launch again uses the latest promoted setup.' : 'This records the saved setup for this deployment. Launch again uses the latest promoted setup for this map.' : 'This preview shows the promoted setup at the time above. The deployment history records the files selected when provisioning begins.'}</p>
<small class="fingerprint" title={recipe.fingerprint}>Recipe {recipe.fingerprint.slice(0, 16)}</small>
</details>
</div>
@@ -0,0 +1,132 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import type { ProvisioningDetail, ProvisioningEvent } from '$lib/provisioning-detail.js';
import { timeFormat } from '$ui/stores/time-zone.js';
import { isProvisioningEventPage } from '$ui/provisioning-presentation.js';
import Button from './primitives/Button.svelte';
let { detail }: { detail: ProvisioningDetail } = $props();
let open = $state(false);
let query = $state('');
let order = $state<'newest' | 'oldest'>('newest');
let appliedQuery = $state('');
let appliedOrder = $state<'newest' | 'oldest'>('newest');
let entries = $state<ProvisioningEvent[]>([]);
let frozenRecent = $state<ProvisioningEvent[] | null>(null);
let total = $state(0);
let baselineTotal = $state<number | null>(null);
let watermark = $state<number | null>(null);
let cursor = $state<number | null>(null);
let loaded = $state(false);
let loading = $state(false);
let error = $state('');
let requestNumber = 0;
let controller: AbortController | null = null;
let disposed = false;
const recent = $derived((frozenRecent ?? detail.events).slice().sort((a, b) => b.id - a.id).slice(0, 5));
const newEvents = $derived(baselineTotal === null ? 0 : Math.max(0, detail.activity.total - baselineTotal));
async function load(more = false) {
if (more && (loading || cursor === null)) return;
controller?.abort();
controller = new AbortController();
const currentController = controller;
const request = ++requestNumber;
const id = detail.deployment.id;
const current = () => !disposed && request === requestNumber && detail.deployment.id === id;
loading = true; error = '';
const requestedQuery = more ? appliedQuery : query;
const requestedOrder = more ? appliedOrder : order;
const params = new URLSearchParams({ order: requestedOrder, limit: '25' });
if (requestedQuery) params.set('query', requestedQuery);
if (more && cursor !== null && watermark !== null) {
params.set('cursor', String(cursor));
params.set('watermark', String(watermark));
}
let timedOut = false;
const timeout = setTimeout(() => { timedOut = true; currentController.abort(); }, 15_000);
try {
const response = await fetch(`/api/provisioning/${encodeURIComponent(id)}/events?${params}`, { signal: currentController.signal, cache: 'no-store' });
if (!response.ok) throw new Error('Could not load activity history.');
const body: unknown = await response.json();
if (!isProvisioningEventPage(body)) throw new Error('The activity response was incomplete.');
if (!current()) return;
entries = more ? [...entries, ...body.events.filter(event => !entries.some(previous => previous.id === event.id))] : body.events;
total = body.total; watermark = body.watermark; cursor = body.nextCursor; loaded = true;
if (!more) { baselineTotal = body.totalEvents; appliedQuery = requestedQuery; appliedOrder = requestedOrder; }
} catch (reason) {
if (current() && (!currentController.signal.aborted || timedOut)) error = timedOut ? 'Activity refresh timed out. Try again.' : reason instanceof Error ? reason.message : 'Could not load activity history.';
} finally {
clearTimeout(timeout);
if (current()) loading = false;
}
}
function toggle(event: Event) {
const expanded = (event.currentTarget as HTMLDetailsElement).open;
open = expanded;
if (expanded) {
frozenRecent = detail.events.slice();
if (!loaded && !loading) void load();
} else frozenRecent = null;
}
function jumpToLatest() {
query = appliedQuery; order = 'newest';
frozenRecent = detail.events.slice();
void load();
}
onDestroy(() => { disposed = true; requestNumber++; controller?.abort(); });
</script>
<section aria-labelledby="activity-title" class="activity">
<div class="heading"><div><span class="eyebrow">Activity</span><h2 id="activity-title">Recent updates</h2></div><span class="count">{detail.activity.total} event{detail.activity.total === 1 ? '' : 's'}</span></div>
<p class="muted">{open ? 'Recent updates stay in place while you read the full history.' : 'Newest first · updates automatically'}</p>
{#if recent.length}<ol class="timeline recent">
{#each recent as event (event.id)}<li><time datetime={event.created_at}>{$timeFormat.dateTime(event.created_at)}</time><p>{event.message}</p></li>{/each}
</ol>{:else}<p class="empty">No detailed activity was recorded. The recorded outcome and timestamps are preserved above.</p>{/if}
<details class="history" ontoggle={toggle}>
<summary>Full activity history <span>{detail.activity.total} events</span></summary>
<div class="history-content">
{#if newEvents > 0}<div class="new-events" role="status"><span>{newEvents} new event{newEvents === 1 ? '' : 's'} since this view was loaded.</span><Button size="sm" onclick={jumpToLatest}>Jump to latest</Button></div>{/if}
<form class="tools" onsubmit={event => { event.preventDefault(); void load(); }}>
<label class="search">Search activity<input type="search" bind:value={query} placeholder="Find a stage or message" /></label>
<label>Order<select bind:value={order} onchange={() => { void load(); }}><option value="newest">Newest first</option><option value="oldest">Oldest first</option></select></label>
<Button type="submit" disabled={loading}>Search</Button>
</form>
{#if error}<div class="history-error" role="alert"><p>{error} {loaded ? 'Your previous history view is still shown.' : ''}</p><Button size="sm" onclick={() => load()}>Retry history</Button></div>{/if}
{#if loaded}<p class="muted">{appliedOrder === 'newest' ? 'Newest first' : 'Oldest first'}{appliedQuery ? ` · Matching “${appliedQuery}”` : ' · All messages'}</p>{/if}
{#if loading && !loaded}<p class="muted" role="status">Loading activity…</p>
{:else if loaded && !entries.length}<p class="empty">{appliedQuery ? 'No activity matches this search.' : 'No detailed activity recorded.'}</p>
{:else}<ol class="timeline">
{#each entries as event (event.id)}<li><time datetime={event.created_at}>{$timeFormat.dateTime(event.created_at)}</time><p>{event.message}</p></li>{/each}
</ol>{/if}
{#if loaded}<div class="footer"><span>Showing {entries.length} of {total} matching events</span>{#if cursor !== null}<Button size="sm" loading={loading} disabled={loading} onclick={() => load(true)}>Show more activity</Button>{/if}</div>{/if}
</div>
</details>
</section>
<style>
.activity { display: grid; gap: var(--space-4); min-width: 0; }
.heading { display: flex; justify-content: space-between; gap: var(--space-4); align-items: end; flex-wrap: wrap; }
.eyebrow { display: block; font-size: var(--text-xs); color: var(--text-tertiary); margin-bottom: var(--space-2); }
h2 { margin: 0; font-size: var(--text-lg); letter-spacing: var(--tracking-tight); }
p { margin: 0; line-height: 1.65; overflow-wrap: anywhere; font-size: var(--text-sm); }
.muted, .count, time, .footer { font-size: var(--text-xs); color: var(--text-secondary); }
.empty { padding: var(--space-4); border: 1px solid var(--border-subtle); border-radius: var(--radius-md); color: var(--text-secondary); }
.timeline { list-style: none; margin: 0 0 0 var(--space-1); padding: 0 0 0 var(--space-5); border-left: 1px solid var(--border-strong); }
.timeline li { padding: 0 0 var(--space-5); position: relative; }.timeline li:last-child { padding-bottom: 0; }
.timeline li::before { content: ''; position: absolute; top: 6px; left: -24px; width: 7px; height: 7px; background: var(--border-control); border-radius: var(--radius-pill); }
.recent li:first-child::before { background: var(--accent-primary); }.timeline p { margin-top: var(--space-1); }
.history { border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); }
summary { cursor: pointer; padding: var(--space-5); font-size: var(--text-sm); font-weight: var(--weight-semibold); overflow-wrap: anywhere; }
summary span { margin-left: var(--space-2); color: var(--text-tertiary); font-size: var(--text-xs); font-weight: var(--weight-regular); }
summary:focus-visible { outline: 2px solid var(--border-focus); outline-offset: -3px; border-radius: var(--radius-sm); }
.history-content { padding: 0 var(--space-5) var(--space-5); display: grid; gap: var(--space-5); }
.tools { display: flex; flex-wrap: wrap; align-items: end; gap: var(--space-3); }.tools label { display: grid; gap: var(--space-2); font-size: var(--text-xs); color: var(--text-secondary); }.search { flex: 1 1 160px; }
input, select { background: var(--surface-sunken); color: var(--text-primary); border: 1px solid var(--border-control); border-radius: var(--radius-sm); padding: 10px; font: inherit; min-height: 40px; width: 100%; }
input:focus-visible, select:focus-visible { outline: 2px solid var(--border-focus); outline-offset: 2px; }
.new-events { display: flex; align-items: center; gap: var(--space-3); flex-wrap: wrap; border: 1px solid var(--border-strong); border-radius: var(--radius-sm); padding: var(--space-3); color: var(--text-secondary); font-size: var(--text-sm); }.new-events span { flex: 1; }
.footer { display: flex; justify-content: space-between; align-items: center; gap: var(--space-3); flex-wrap: wrap; line-height: 1.6; }
.history-error { display: grid; justify-items: start; gap: var(--space-3); color: var(--state-danger); }
@media (max-width: 600px) { summary { padding: var(--space-4); }.history-content { padding: 0 var(--space-4) var(--space-4); }.tools { align-items: stretch; }.tools .search { flex-basis: 100%; } }
</style>
@@ -0,0 +1,123 @@
<script lang="ts">
import type { ProvisioningDetail } from '$lib/provisioning-detail.js';
import { provisioningGroups, progressIcons, progressStateLabels } from '$ui/provisioning-presentation.js';
import { timeFormat } from '$ui/stores/time-zone.js';
import Badge from './primitives/Badge.svelte';
let { detail }: { detail: ProvisioningDetail } = $props();
const groups = $derived(provisioningGroups(detail));
const progress = $derived(detail.progress);
let expanded = $state<Record<string, boolean>>({});
const evidenceLabels = { pass: 'Passed', waiting: 'Waiting', fail: 'Failed', unknown: 'Unknown' };
const sources = { komodo: 'Komodo', rcon: 'RCON', telemetry: 'Telemetry', combined: 'RCON and telemetry' };
</script>
<section class="execution" aria-labelledby="execution-title">
<div class="section-heading"><div><span class="eyebrow">Execution</span><h2 id="execution-title">Progress and evidence</h2></div></div>
{#if !progress}
<div class="legacy"><h3>{detail.deployment.stage}</h3><p>Detailed checkpoints were not recorded for this operation. Its recorded outcome and original activity remain available below.</p></div>
{:else}
<ol class="phases">
{#each groups as group (group.key)}
<li class:current={group.state === 'current' || group.state === 'waiting'} class:failed={group.state === 'failed'}>
<details open={expanded[group.key] ?? ['current', 'waiting', 'failed'].includes(group.state)}>
<summary
onclick={event => { event.preventDefault(); expanded[group.key] = !(expanded[group.key] ?? ['current', 'waiting', 'failed'].includes(group.state)); }}
onkeydown={event => {
if (event.key !== 'Enter' && event.key !== ' ') return;
event.preventDefault();
if (!event.repeat) expanded[group.key] = !(expanded[group.key] ?? ['current', 'waiting', 'failed'].includes(group.state));
}}
>
<span class="phase-icon state-{group.state}" aria-hidden="true">{progressIcons[group.state]}</span>
<span class="phase-title">{group.label}</span>
<span class="phase-state state-{group.state}">{progressStateLabels[group.state]}</span>
</summary>
<ol class="steps">
{#each group.steps as step (step.key)}
<li aria-current={step.state === 'current' || step.state === 'waiting' ? 'step' : undefined}>
<span class="step-icon state-{step.state}" aria-hidden="true">{progressIcons[step.state]}</span>
<div><strong>{step.label}</strong><span class="step-meta state-{step.state}">{progressStateLabels[step.state]}{#if step.at} · <time datetime={step.at}>{$timeFormat.dateTime(step.at)}</time>{/if}</span></div>
</li>
{/each}
</ol>
</details>
</li>
{/each}
</ol>
{#if progress.command}
<div class="command" class:uncertain={progress.command.state === 'uncertain'}>
<h3>{progress.command.state === 'attempted' ? 'Command submission attempted' : progress.command.state === 'accepted' ? 'Remote request accepted' : progress.command.state === 'rejected' ? 'Remote request rejected' : 'Remote response uncertain'}</h3>
<p>{progress.command.message}</p>
<time datetime={progress.command.at}>{$timeFormat.dateTime(progress.command.at)}</time>
</div>
{/if}
{#if progress.checkpoints.some(checkpoint => !groups.some(group => group.steps.some(step => step.key === checkpoint.key)))}
<details class="milestones"><summary>Recorded checkpoints</summary><ul>
{#each progress.checkpoints.filter(checkpoint => !groups.some(group => group.steps.some(step => step.key === checkpoint.key))) as checkpoint (checkpoint.key)}
<li><strong>{checkpoint.label}</strong><time datetime={checkpoint.at}>{$timeFormat.dateTime(checkpoint.at)}</time></li>
{/each}
</ul></details>
{/if}
<div class="evidence-heading"><h3>Latest recorded checks</h3>{#if progress.observedAt}<span>Evaluated <time datetime={progress.observedAt}>{$timeFormat.dateTime(progress.observedAt)}</time></span>{/if}</div>
{#if progress.checks.length}
<div class="checks">
{#each progress.checks as check (check.key)}
<article class="check">
<div class="check-heading"><strong>{check.label}</strong><Badge variant={check.result === 'pass' ? 'online' : check.result === 'fail' ? 'danger' : check.result === 'waiting' ? 'warning' : 'neutral'}>{evidenceLabels[check.result]}</Badge></div>
{#if check.expected !== null || check.actual !== null}<dl>
{#if check.expected !== null}<div><dt>Expected</dt><dd>{check.expected}</dd></div>{/if}
<div><dt>Observed</dt><dd>{check.actual ?? 'Unknown'}</dd></div>
</dl>{/if}
{#if check.message}<p>{check.message}</p>{/if}
<div class="evidence-meta"><span>{check.source ? sources[check.source] : 'Source not recorded'}</span><span>Checked <time datetime={check.checkedAt}>{$timeFormat.dateTime(check.checkedAt)}</time></span>{#if check.receivedAt}<span>Evidence received <time datetime={check.receivedAt}>{$timeFormat.dateTime(check.receivedAt)}</time></span>{/if}</div>
</article>
{/each}
</div>
{:else}<p class="empty">No check evidence has been recorded for this operation yet.</p>{/if}
{#if detail.deployment.expected_map}<p class="map-note">Selected engine map: <code>{detail.deployment.expected_map}</code>. Engine-map verification does not independently verify a variant or start location.</p>{/if}
{/if}
</section>
<style>
.execution { display: grid; gap: var(--space-5); min-width: 0; }
h2, h3, p { margin: 0; }
h2 { font-size: var(--text-lg); letter-spacing: var(--tracking-tight); }
h3 { font-size: var(--text-sm); font-weight: var(--weight-semibold); }
.eyebrow { display: block; margin-bottom: var(--space-2); font-size: var(--text-xs); color: var(--text-tertiary); }
.phases, .steps, .milestones ul { list-style: none; padding: 0; margin: 0; }
.phases { display: grid; gap: var(--space-3); }
.phases > li { border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); overflow: hidden; }
.phases > li.current { border-color: color-mix(in srgb, var(--accent-primary) 60%, var(--border-subtle)); }
.phases > li.failed { border-color: color-mix(in srgb, var(--state-danger) 60%, var(--border-subtle)); }
summary { cursor: pointer; }
.phases summary { display: flex; align-items: center; gap: var(--space-3); padding: var(--space-5); list-style: none; }
.phases summary::-webkit-details-marker { display: none; }
.phases summary::after { content: '+'; color: var(--text-tertiary); font-size: 18px; }
.phases details[open] > summary::after { content: '−'; }
summary:focus-visible { outline: 2px solid var(--border-focus); outline-offset: -3px; border-radius: var(--radius-sm); }
.phase-icon { flex: 0 0 26px; display: grid; place-items: center; width: 26px; height: 26px; border: 1px solid var(--border-strong); border-radius: var(--radius-pill); font-size: var(--text-sm); }
.phase-title { flex: 1; font-size: var(--text-sm); font-weight: var(--weight-semibold); }
.phase-state { font-size: var(--text-xs); }
.steps { border-top: 1px solid var(--border-subtle); padding: var(--space-4) var(--space-5); display: grid; gap: var(--space-4); }
.steps li { display: flex; align-items: flex-start; gap: var(--space-3); }
.step-icon { flex: 0 0 26px; text-align: center; line-height: 1.5; }
.steps strong { display: block; font-size: var(--text-sm); font-weight: var(--weight-medium); line-height: 1.5; }
.step-meta { display: block; font-size: var(--text-xs); margin-top: var(--space-1); line-height: 1.6; }
.state-current { color: var(--accent-primary-hover); }.state-waiting { color: var(--state-warning); }.state-failed { color: var(--state-danger); }.state-done { color: var(--state-online); }
.state-unknown, .state-pending, .state-cancelled, time { color: var(--text-secondary); }
.command, .legacy { display: grid; gap: var(--space-2); padding: var(--space-4); border: 1px solid var(--border-subtle); border-radius: var(--radius-md); }
.command.uncertain { border-color: color-mix(in srgb, var(--state-warning) 45%, var(--border-subtle)); }
p { color: var(--text-secondary); font-size: var(--text-sm); line-height: 1.65; overflow-wrap: anywhere; }
time, .evidence-meta, .evidence-heading span, .milestones { font-size: var(--text-xs); }
.evidence-heading { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: var(--space-2); padding-top: var(--space-2); }
.evidence-heading span { color: var(--text-secondary); }
.checks { border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); overflow: hidden; }
.check { padding: var(--space-5); display: grid; gap: var(--space-3); }.check + .check { border-top: 1px solid var(--border-subtle); }
.check-heading { display: flex; gap: var(--space-3); align-items: center; justify-content: space-between; flex-wrap: wrap; }.check-heading strong { font-size: var(--text-sm); }
dl { display: flex; gap: var(--space-4) var(--space-8); margin: 0; flex-wrap: wrap; }dt { font-size: var(--text-xs); color: var(--text-tertiary); }dd { margin: var(--space-1) 0 0; font-size: var(--text-sm); overflow-wrap: anywhere; }
.evidence-meta { display: flex; gap: var(--space-1) var(--space-4); flex-wrap: wrap; color: var(--text-tertiary); line-height: 1.6; }
.milestones { border-block: 1px solid var(--border-subtle); padding: var(--space-3) 0; }.milestones summary { color: var(--text-secondary); }.milestones ul { display: grid; gap: var(--space-3); padding-top: var(--space-4); }.milestones li { display: grid; gap: var(--space-1); }.milestones strong { font-weight: var(--weight-medium); }
.map-note { font-size: var(--text-xs); }code { font-family: var(--font-mono); overflow-wrap: anywhere; }
@media (max-width: 600px) { .phases summary { padding: var(--space-4); gap: var(--space-2); flex-wrap: wrap; }.phase-title { flex-basis: calc(100% - 70px); }.phase-state { padding-left: 34px; order: 1; flex-basis: 100%; }.phases summary::after { order: 0; }.steps, .check { padding: var(--space-4); } }
</style>
+202
View File
@@ -0,0 +1,202 @@
import { provisioningPhases, type ProvisioningDetail, type ProvisioningEvent, type ProvisioningEventPage, type ProvisioningPhase } from '$lib/provisioning-detail.js';
import { actionCompleted, actionLabels, provisioningActions, type ProvisioningAction } from '$lib/provisioning.js';
export const activeProvisioningStates = ['queued', 'waiting', 'running', 'awaiting-health'];
export type ProgressState = 'done' | 'current' | 'waiting' | 'failed' | 'pending' | 'cancelled' | 'unknown';
export interface ProgressStep { key: ProvisioningPhase; label: string; state: ProgressState; at: string | null }
export interface ProgressGroup { key: string; label: string; steps: ProgressStep[]; state: ProgressState }
const phaseLabels: Record<ProvisioningPhase, string> = {
requested: 'Request recorded', eligibility: 'Check eligibility and capacity', 'startup-queue': 'Wait for a startup slot',
prepare: 'Prepare server files', 'create-deployment': 'Create Komodo deployment', 'container-action': 'Start container',
'remove-container': 'Stop and remove container', 'delete-deployment': 'Delete Komodo deployment', cleanup: 'Remove files and registration',
'verify-container': 'Confirm container readiness', 'verify-game': 'Verify game status', 'verify-map': 'Verify selected engine map',
'verify-stopped': 'Confirm container is stopped'
};
type GroupDefinition = { key: string; label: string; phases: ProvisioningPhase[] };
const request: GroupDefinition = { key: 'request', label: 'Request and eligibility', phases: ['requested', 'eligibility'] };
const creation: GroupDefinition = { key: 'create', label: 'Create and start', phases: ['startup-queue', 'prepare', 'create-deployment', 'container-action'] };
const removal: GroupDefinition = { key: 'remove', label: 'Remove the instance', phases: ['remove-container', 'delete-deployment', 'cleanup'] };
const readiness: GroupDefinition = { key: 'verify', label: 'Verify readiness', phases: ['verify-container', 'verify-game', 'verify-map'] };
function definitions(action: ProvisioningAction): GroupDefinition[] {
if (action === 'create') return [{ ...request, phases: ['requested'] }, creation, readiness];
if (action === 'rebuild') return [request, removal, { ...creation, label: 'Create a fresh instance' }, readiness];
if (action === 'destroy') return [request, removal];
return [request, { key: 'command', label: `${actionLabels[action]} the existing container`, phases: ['container-action'] },
{ ...readiness, phases: action === 'stop' ? ['verify-stopped'] : ['verify-container', 'verify-game'] }];
}
export function provisioningGroups(detail: ProvisioningDetail): ProgressGroup[] {
const progress = detail.progress;
if (!progress) return [];
const groups = definitions(detail.deployment.action);
const phases = groups.flatMap(group => group.phases);
const current = progress.phase ? phases.indexOf(progress.phase) : -1;
const active = activeProvisioningStates.includes(detail.deployment.state);
return groups.map(group => {
const steps = group.phases.map((key): ProgressStep => {
const checkpoint = progress.checkpoints.find(item => item.key === key);
const checkKey = key === 'verify-container' ? 'container' : key === 'verify-game' ? 'game' : key === 'verify-map' ? 'map' : key === 'verify-stopped' ? 'stopped' : null;
const check = checkKey ? progress.checks.find(item => item.key === checkKey) : null;
const index = phases.indexOf(key);
const isCurrent = key === progress.phase;
const state: ProgressState = check ? ({ pass: 'done', fail: 'failed', waiting: 'waiting', unknown: 'unknown' } as const)[check.result]
: checkpoint ? 'done'
: isCurrent && ['failed', 'interrupted'].includes(detail.deployment.state) ? 'failed'
: isCurrent && active ? progress.blocker || ['waiting', 'queued'].includes(detail.deployment.state) ? 'waiting' : 'current'
: current >= 0 && index >= current && detail.deployment.state === 'cancelled' ? 'cancelled'
: active && current >= 0 && index > current ? 'pending' : 'unknown';
const label = key === 'container-action' && ['start', 'restart', 'stop'].includes(detail.deployment.action)
? `${actionLabels[detail.deployment.action]} existing container` : phaseLabels[key];
return { key, label, state, at: check?.checkedAt ?? checkpoint?.at ?? (isCurrent ? progress.phaseStartedAt : null) };
});
const states = steps.map(step => step.state);
const state = (['failed', 'current', 'waiting'] as const).find(value => states.includes(value))
?? (states.every(value => value === 'done') ? 'done' : states.includes('unknown') ? 'unknown' : states.includes('cancelled') ? 'cancelled' : 'pending');
return { key: group.key, label: group.label, steps, state };
});
}
export const progressStateLabels: Record<ProgressState, string> = {
done: 'Completed', current: 'In progress', waiting: 'Waiting', failed: 'Needs attention', pending: 'Not started', cancelled: 'Not run', unknown: 'Not recorded'
};
export const progressIcons: Record<ProgressState, string> = { done: '✓', current: '•', waiting: '◷', failed: '!', pending: '○', cancelled: '–', unknown: '?' };
export function provisioningOutcome(detail: ProvisioningDetail) {
const { deployment: operation, progress } = detail;
const active = activeProvisioningStates.includes(operation.state);
const failure = ['failed', 'interrupted'].includes(operation.state);
const headline = operation.state === 'done' ? actionCompleted[operation.action]
: operation.state === 'cancelled' ? `${actionLabels[operation.action]} request cancelled`
: failure ? `${actionLabels[operation.action]} ${operation.state === 'interrupted' ? 'interrupted' : 'failed'}`
: operation.state === 'warning' ? `${actionLabels[operation.action]} completed with warnings`
: progress?.blocker?.message ?? operation.stage;
let description: string;
if (detail.canResumeCleanup) description = 'The remote deployment was deleted. Resume the remaining local cleanup without repeating container removal.';
else if (detail.canResumeRebuild) description = 'The remote deployment was deleted. Resume local cleanup, then create a fresh instance in the reserved server slot.';
else if (detail.canResumeVerification) description = 'Resume observation of the existing instance. This does not create, restart, or rebuild a container.';
else if (operation.state === 'cancelled') description = 'This request is cancelled. Recorded prior work is preserved below; remaining work will not run.';
else if (failure) description = progress?.command?.state === 'uncertain'
? 'The remote response was uncertain. Inspect the recorded result and server before requesting another operation.'
: 'Review the recorded failure and execution evidence before submitting another operation.';
else if (operation.state === 'warning') description = 'The operation finished. Review the recorded warnings below for any remaining issue.';
else if (operation.state === 'done') description = !progress
? 'This operation was recorded as completed. Detailed verification evidence was not recorded.'
: operation.action === 'destroy' ? 'The instance removal completed. This record preserves its results after deletion.'
: operation.action === 'stop' ? 'The container was confirmed stopped. Its files and registration were retained.'
: 'The operation completed its readiness checks. These recorded results describe that attempt; current health is available on the server page.';
else if (active && progress?.phase === 'cleanup' && !progress.blocker) description = operation.action === 'rebuild'
? 'Remote deployment deletion was confirmed. Local cleanup must finish before a fresh instance is created in the reserved slot.'
: 'Remote deployment deletion was confirmed. The remaining local files and registration cleanup will continue without repeating container removal.';
else if (progress?.blocker) description = progress.blocker.automatic
? 'The worker will recheck this condition and continue automatically when eligible.'
: 'Review the blocking condition below before this operation can continue.';
else if (operation.state === 'queued') description = 'The request is saved. It will start when its dependencies and startup capacity allow.';
else if (operation.state === 'waiting') description = 'Waiting for the worker’s eligibility checks. The recorded stage below describes the last known condition.';
else description = operation.action === 'destroy' ? 'Removal is in progress. Confirmed checkpoints and remaining cleanup are recorded below.'
: operation.action === 'rebuild' ? 'The old instance is removed before a fresh instance is created in its reserved slot. Follow each checkpoint below.'
: 'The worker is executing this request and checking its resulting state.';
return { active, headline, description };
}
/** Accept return context only from the local operation surfaces we expose. */
export function provisioningReturn(value: string | null | undefined): { href: string; label: string } {
const fallback = { href: '/provisioning', label: 'Back to Provisioning' };
if (!value || !value.startsWith('/') || value.startsWith('//') || value.includes('\\')) return fallback;
try {
const url = new URL(value, 'https://electric-cherry.invalid');
if (url.origin !== 'https://electric-cherry.invalid') return fallback;
if (url.pathname === '/provisioning') {
const params = new URLSearchParams();
for (const key of ['view', 'action', 'status', 'server', 'limit']) {
const item = url.searchParams.get(key);
if (item) params.set(key, item);
}
return { href: '/provisioning' + (params.size ? `?${params}` : ''), label: 'Back to Provisioning' };
}
const match = url.pathname.match(/^\/(campaigns|betas|maintenance|collections\/launches)\/[^/]+$/);
if (match) return { href: url.pathname, label: `Back to ${match[1] === 'collections/launches' ? 'collection' : match[1] === 'campaigns' ? 'campaign' : match[1] === 'betas' ? 'beta' : 'maintenance'}` };
} catch { /* Fall back to the local list. */ }
return fallback;
}
export function elapsedTime(start: string | null | undefined, end: number): string {
if (!start || !Number.isFinite(Date.parse(start))) return 'Not recorded';
const seconds = Math.max(0, Math.floor((end - Date.parse(start)) / 1000));
if (seconds < 60) return `${seconds}s`;
if (seconds < 3600) return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
return `${Math.floor(seconds / 3600)}h ${Math.floor(seconds % 3600 / 60)}m`;
}
const record = (value: unknown): value is Record<string, unknown> => !!value && typeof value === 'object' && !Array.isArray(value);
const text = (value: unknown): value is string => typeof value === 'string';
const number = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value);
const nullableText = (value: unknown) => value === null || text(value);
const timestamp = (value: unknown): value is string => text(value) && Number.isFinite(Date.parse(value));
const nullableTime = (value: unknown) => value === null || timestamp(value);
const scalar = (value: unknown) => value === null || text(value) || number(value);
const localLink = (value: unknown) => value === null || (text(value) && value.startsWith('/') && !value.startsWith('//') && !value.includes('\\'));
const optionalLink = (value: unknown) => value === undefined || localLink(value);
const phase = (value: unknown) => value === null || provisioningPhases.includes(value as ProvisioningPhase);
const event = (value: unknown): value is ProvisioningEvent => record(value) && number(value.id) && timestamp(value.created_at) && text(value.message);
export function isProvisioningEventPage(value: unknown): value is ProvisioningEventPage {
return record(value) && Array.isArray(value.events) && value.events.every(event)
&& number(value.total) && number(value.totalEvents) && (value.latestId === null || number(value.latestId))
&& number(value.watermark) && (value.nextCursor === null || number(value.nextCursor));
}
/** Validate every nested shape consumed by the page before replacing its last good snapshot. */
export function isProvisioningDetail(value: unknown, id: string): value is ProvisioningDetail {
if (!record(value) || !record(value.deployment)) return false;
const operation = value.deployment;
if (operation.id !== id || !provisioningActions.includes(operation.action as ProvisioningAction)
|| !['server', 'state', 'stage', 'triggered_by', 'warnings'].every(key => text(operation[key]))
|| !timestamp(operation.started_at) || !nullableTime(operation.finished_at) || !nullableTime(operation.server_deleted_at)
|| !nullableText(operation.error) || !number(operation.server_exists)
|| (operation.submitted_at !== undefined && !nullableTime(operation.submitted_at))
|| (operation.expected_map !== undefined && !nullableText(operation.expected_map))) return false;
if (!timestamp(value.generatedAt) || !Array.isArray(value.events) || !value.events.every(event)
|| !Array.isArray(value.warningMessages) || !value.warningMessages.every(text)
|| !['canResumeVerification', 'canResumeCleanup', 'canResumeRebuild', 'canCancelRemoval'].every(key => typeof value[key] === 'boolean')
|| !nullableText(value.blueprintName)) return false;
if (!record(value.activity) || !number(value.activity.total) || !(value.activity.latestId === null || number(value.activity.latestId))
|| !(value.activity.failureExcerpt === null || event(value.activity.failureExcerpt))) return false;
if (!Array.isArray(value.sources) || !value.sources.every(source => record(source)
&& ['campaign', 'beta', 'maintenance', 'watchdog'].includes(source.kind as string)
&& text(source.source_id) && text(source.label) && localLink(source.href))) return false;
if (!(value.collection === null || record(value.collection) && text(value.collection.id) && text(value.collection.name))) return false;
if (!record(value.capacity) || !timestamp(value.capacity.evaluatedAt) || !Array.isArray(value.capacity.pools)
|| !value.capacity.pools.every(pool => record(pool) && ['startup', 'restart'].includes(pool.kind as string)
&& number(pool.limit) && number(pool.used) && typeof pool.ownsSlot === 'boolean' && (pool.requestOrder === null || number(pool.requestOrder))
&& Array.isArray(pool.holders) && pool.holders.every(holder => record(holder) && nullableText(holder.id) && text(holder.server)))) return false;
if (value.recipe !== null) {
const recipe = value.recipe;
if (!record(recipe) || !['t5', 't6'].includes(recipe.game as string) || !timestamp(recipe.capturedAt)
|| !['mapSlug', 'mapName', 'expectedMap', 'image', 'configFile', 'configSource', 'fingerprint'].every(key => text(recipe[key]))
|| !Array.isArray(recipe.files) || !recipe.files.every(file => record(file)
&& ['name', 'kind', 'path', 'sha256'].every(key => text(file[key])) && number(file.size) && (file.version === null || number(file.version)))) return false;
}
if (value.progress !== null) {
const progress = value.progress;
if (!record(progress) || progress.version !== 1 || !phase(progress.phase) || !nullableTime(progress.phaseStartedAt) || !nullableTime(progress.observedAt)
|| ![null, 'queued', 'attempted'].includes(progress.recipeStatus as null | string)) return false;
if (!(progress.blocker === null || record(progress.blocker) && text(progress.blocker.code) && text(progress.blocker.message)
&& typeof progress.blocker.automatic === 'boolean' && optionalLink(progress.blocker.href)
&& (progress.blocker.label === undefined || nullableText(progress.blocker.label)))) return false;
if (!Array.isArray(progress.checks) || !progress.checks.every(check => record(check)
&& ['container', 'game', 'map', 'occupancy', 'stopped'].includes(check.key as string)
&& ['pass', 'waiting', 'fail', 'unknown'].includes(check.result as string)
&& [null, 'komodo', 'rcon', 'telemetry', 'combined'].includes(check.source as null | string)
&& text(check.label) && timestamp(check.checkedAt) && nullableTime(check.receivedAt)
&& scalar(check.expected) && scalar(check.actual) && nullableText(check.message))) return false;
if (!Array.isArray(progress.checkpoints) || !progress.checkpoints.every(checkpoint => record(checkpoint)
&& text(checkpoint.key) && text(checkpoint.label) && timestamp(checkpoint.at) && phase(checkpoint.phase))) return false;
if (!(progress.command === null || record(progress.command) && ['attempted', 'accepted', 'rejected', 'uncertain'].includes(progress.command.state as string)
&& timestamp(progress.command.at) && text(progress.command.message))) return false;
if (!(progress.recovery === null || record(progress.recovery) && ['held', 'released', 'transferred', 'unknown'].includes(progress.recovery.state as string)
&& timestamp(progress.recovery.at) && text(progress.recovery.message)
&& (progress.recovery.operationId === undefined || nullableText(progress.recovery.operationId)))) return false;
}
return true;
}
+131
View File
@@ -0,0 +1,131 @@
export type ProvisioningMutation = 'resume' | 'cancel';
export interface ProvisioningRefreshState<T> {
id: string;
detail: T | null;
refreshing: boolean;
busy: ProvisioningMutation | null;
loadError: string;
actionError: string;
notice: string;
refreshedAt: string | null;
}
interface Options<T> {
onChange: (state: ProvisioningRefreshState<T>) => void;
validate: (value: unknown, id: string) => value is T;
fetcher?: typeof fetch;
now?: () => string;
timeoutMs?: number;
}
/** Reads and action feedback belong to a particular visit, even when its ID is revisited. */
export function createProvisioningRefresh<T>(options: Options<T>) {
const request = options.fetcher ?? fetch;
const now = options.now ?? (() => new Date().toISOString());
let visit = 0;
let read = 0;
let disposed = false;
let pending: { controller: AbortController; promise: Promise<void>; timer?: ReturnType<typeof setTimeout> } | null = null;
let mutation: AbortController | null = null;
let state: ProvisioningRefreshState<T> = {
id: '', detail: null, refreshing: false, busy: null, loadError: '', actionError: '', notice: '', refreshedAt: null
};
const publish = (patch: Partial<ProvisioningRefreshState<T>>) => {
state = { ...state, ...patch };
if (!disposed) options.onChange(state);
};
function invalidateRead() {
read++;
if (pending) {
clearTimeout(pending.timer);
pending.controller.abort();
pending = null;
}
}
async function load(force = false, reconcile = false): Promise<void> {
if (disposed || !state.id || (state.busy && !reconcile)) return;
if (pending && !force) return pending.promise;
invalidateRead();
const currentVisit = visit;
const currentRead = read;
const id = state.id;
const current = () => !disposed && currentVisit === visit && currentRead === read && id === state.id;
const controller = new AbortController();
const entry = { controller, promise: Promise.resolve(), timer: undefined as ReturnType<typeof setTimeout> | undefined };
let timedOut = false;
pending = entry;
publish({ refreshing: true });
entry.timer = setTimeout(() => { timedOut = true; controller.abort(); }, options.timeoutMs ?? 15_000);
entry.promise = (async () => {
try {
const response = await request(`/api/provisioning/${encodeURIComponent(id)}?events=recent`, { signal: controller.signal, cache: 'no-store' });
if (!response.ok) throw new Error(response.status === 404 ? 'Provisioning operation not found.' : 'Could not refresh this operation.');
const detail: unknown = await response.json();
if (!options.validate(detail, id)) throw new Error('The operation response was incomplete. Try refreshing again.');
if (current()) publish({ detail, loadError: '', refreshedAt: now() });
} catch (error) {
if (current() && (!controller.signal.aborted || timedOut)) publish({
loadError: timedOut ? 'Operation refresh timed out. Try again.' : error instanceof Error ? error.message : 'Could not refresh this operation.'
});
} finally {
clearTimeout(entry.timer);
if (pending === entry) pending = null;
if (current()) publish({ refreshing: false });
}
})();
return entry.promise;
}
function setOperation(id: string, detail: T | null = null, refreshedAt: string | null = null) {
if (disposed) return;
visit++;
invalidateRead();
mutation?.abort();
mutation = null;
publish({ id, detail, refreshedAt, refreshing: false, busy: null, loadError: '', actionError: '', notice: '' });
if (!detail) void load();
}
async function act(action: ProvisioningMutation, successMessage: string): Promise<void> {
if (disposed || !state.id || state.busy) return;
const currentVisit = visit;
const id = state.id;
const current = () => !disposed && currentVisit === visit && id === state.id;
invalidateRead();
const controller = new AbortController();
mutation = controller;
let timedOut = false;
const timer = setTimeout(() => { timedOut = true; controller.abort(); }, options.timeoutMs ?? 15_000);
publish({ busy: action, refreshing: false, actionError: '', notice: '' });
try {
const response = await request(`/api/provisioning/${encodeURIComponent(id)}/${action}`, { method: 'POST', signal: controller.signal });
const body = await response.json() as { error?: string };
if (!response.ok) throw new Error(body.error ?? 'The operation changed before this request could be accepted.');
if (current()) publish({ notice: successMessage });
} catch (error) {
if (current() && (!controller.signal.aborted || timedOut)) publish({ actionError: timedOut
? 'The request timed out and may have reached the server. Review the refreshed operation before trying again.'
: error instanceof Error ? error.message : 'Could not confirm the request. Review the operation before trying again.' });
} finally {
clearTimeout(timer);
if (mutation === controller) mutation = null;
// A failed reconciliation is a read error, never a failed accepted mutation.
if (current()) {
await load(true, true);
if (current()) publish({ busy: null });
}
}
}
function dispose() {
disposed = true;
visit++;
invalidateRead();
mutation?.abort();
mutation = null;
}
return { setOperation, refresh: () => load(), forceRefresh: () => load(true), act, dispose };
}
+2 -2
View File
@@ -2,8 +2,8 @@ import { json, error } from '@sveltejs/kit';
import { db } from '$lib/server/db/index.js';
import { getDeployment } from '$lib/server/provisioning/index.js';
export async function GET({ params }: { params: { id: string } }) {
const lifecycle = getDeployment(params.id);
export async function GET({ params, url }: { params: { id: string }; url?: URL }) {
const lifecycle = getDeployment(params.id, { events: url?.searchParams.get('events') === 'recent' ? 'recent' : 'all' });
if (lifecycle) return json(lifecycle);
const dep = db()
.prepare('SELECT * FROM deployments WHERE id = ?')
@@ -0,0 +1,24 @@
import { json, error } from '@sveltejs/kit';
import { db } from '$lib/server/db/index.js';
import { deploymentActivity } from '$lib/server/provisioning/index.js';
function integer(params: URLSearchParams, key: string): number | undefined {
const raw = params.get(key);
if (raw === null) return undefined;
if (!/^\d+$/.test(raw)) error(400, `${key} must be a nonnegative integer.`);
const value = Number(raw);
if (!Number.isSafeInteger(value)) error(400, `${key} must be a nonnegative integer.`);
return value;
}
export function GET({ params, url }: { params: { id: string }; url: URL }) {
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ?').get(params.id)) error(404, 'Provisioning operation not found.');
const order = url.searchParams.get('order') ?? 'newest';
if (order !== 'newest' && order !== 'oldest') error(400, 'Choose newest or oldest activity order.');
const query = url.searchParams.get('query') ?? '';
if (query.length > 500) error(400, 'Activity search must be 500 characters or fewer.');
const limit = integer(url.searchParams, 'limit');
if (limit === 0) error(400, 'Activity limit must be at least 1.');
return json(deploymentActivity(params.id, { order, query, limit,
cursor: integer(url.searchParams, 'cursor'), watermark: integer(url.searchParams, 'watermark') }));
}
+1 -1
View File
@@ -128,7 +128,7 @@
</div>
{#snippet row(d: Deployment)}
<article class="deployment-row"><a class="row-link" href="/provisioning/{d.id}">
<article class="deployment-row"><a class="row-link" href={`/provisioning/${encodeURIComponent(d.id)}?returnTo=${encodeURIComponent($page.url.pathname + $page.url.search)}`}>
<div class="row-head">
<div><Badge variant={d.action === 'create' ? 'info' : 'neutral'}>{actionLabels[d.action]}</Badge>
<strong class="server"><HistoricalServerIdentity name={d.server} deletedAt={d.server_deleted_at} /></strong>
+2 -2
View File
@@ -2,8 +2,8 @@ import { redirect, error } from '@sveltejs/kit';
import { getDeployment } from '$lib/server/provisioning/index.js';
import { db } from '$lib/server/db/index.js';
export function load({ params }: { params: { id: string } }) {
const detail = getDeployment(params.id);
if (detail) return {};
const detail = getDeployment(params.id, { events: 'recent' });
if (detail) return { detail };
if (db().prepare('SELECT id FROM deployments WHERE id = ?').get(params.id)) {
redirect(307, '/campaigns/' + encodeURIComponent(params.id));
}
+156 -155
View File
@@ -1,177 +1,178 @@
<script lang="ts">
import { onDestroy, onMount, untrack } from 'svelte';
import { page } from '$app/stores';
import type { PageData } from './$types';
import type { ProvisioningDetail } from '$lib/provisioning-detail.js';
import { actionLabels } from '$lib/provisioning.js';
import { timeFormat } from '$ui/stores/time-zone.js';
import { onMount } from 'svelte';
import { pollVisible } from '$ui/poll.js';
import { serverTitle } from '$ui/server-labels.js';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
import OperationHistory from '$ui/components/OperationHistory.svelte';
import type { OperationStep } from '$lib/operation-flow.js';
import { page } from '$app/stores';
import { toast } from '$ui/toast.js';
import { createProvisioningRefresh, type ProvisioningRefreshState } from '$ui/provisioning-refresh.js';
import { elapsedTime, isProvisioningDetail, provisioningOutcome, provisioningReturn } from '$ui/provisioning-presentation.js';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
import Button from '$ui/components/primitives/Button.svelte';
import DeployServerDialog from '$ui/components/DeployServerDialog.svelte';
import Card from '$ui/components/primitives/Card.svelte';
import CopyValue from '$ui/components/CopyValue.svelte';
import ProvisioningProgress from '$ui/components/ProvisioningProgress.svelte';
import ProvisioningActivity from '$ui/components/ProvisioningActivity.svelte';
import LaunchRecipe from '$ui/components/LaunchRecipe.svelte';
import type { LaunchRecipeSummary } from '$lib/deployment-plan.js';
import DeployServerDialog from '$ui/components/DeployServerDialog.svelte';
import { actionLabels, actionCompleted, type ProvisioningAction } from '$lib/provisioning.js';
interface Deployment {
id: string; expected_map?: string; server: string; server_deleted_at?: string | null; action: ProvisioningAction; state: string;
stage: string; started_at: string; finished_at: string | null;
triggered_by: string; error: string | null; warnings: string; server_exists: number;
}
interface Detail { deployment: Deployment; events: { id: number; created_at: string; message: string }[]; sources: { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string }[]; recipe: LaunchRecipeSummary | null; blueprintName: string | null; canResumeVerification: boolean; canResumeCleanup: boolean; canResumeRebuild: boolean; canCancelRemoval: boolean; collection: { id: string; name: string } | null }
let detail = $state<Detail | null>(null);
let error = $state('');
let refreshing = false;
let now = $state(Date.now());
let open = $state(false);
let resuming = $state(false);
let deployment = $derived(detail?.deployment);
let active = $derived(!deployment?.server_deleted_at && ['queued', 'awaiting-health', 'waiting', 'running'].includes(deployment?.state ?? ''));
let warnings = $derived.by(() => {
try { return JSON.parse(deployment?.warnings ?? '[]') as string[]; } catch { return []; }
let { data }: { data: PageData } = $props();
const initial = untrack(() => data.detail);
let view = $state<ProvisioningRefreshState<ProvisioningDetail>>({
id: initial.deployment.id, detail: initial, refreshing: false, busy: null,
loadError: '', actionError: '', notice: '', refreshedAt: initial.generatedAt
});
const createSteps = ['Prepare server files', 'Create Komodo deployment', 'Start container', 'Verify Komodo health', 'Verify RCON health', 'Verify selected map'];
const createStages = ['Preparing server files', 'Creating Komodo deployment', 'Starting container', 'Waiting for Komodo health', 'Waiting for RCON health', 'Waiting for selected map'];
const destroySteps = ['Wait for eligibility', 'Stop and remove container', 'Delete Komodo deployment', 'Remove server files and registry'];
const destroyStages = ['Waiting to verify', 'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'];
let steps = $derived(deployment?.action === 'destroy' ? ['Request destruction', ...destroySteps]
: deployment?.action === 'rebuild' ? ['Request rebuild', ...destroySteps, 'Queue fresh instance', ...createSteps]
: deployment?.action === 'create' ? ['Request creation', ...createSteps]
: ['Request operation', 'Wait for eligibility', `${deployment ? actionLabels[deployment.action] : 'Start'} container`, deployment?.action === 'stop' ? 'Verify container stopped' : 'Verify container and RCON health']);
let currentStep = $derived.by(() => {
if (!deployment) return 0;
if (deployment.state === 'done' || deployment.state === 'warning') return steps.length;
const { action, state, stage } = deployment;
if (state === 'waiting' && stage !== 'Removing server files and registry') return 1;
if (action === 'start' || action === 'restart' || action === 'stop') return state === 'awaiting-health' ? 3 : state === 'running' ? 2 : stage.includes('container') ? 2 : 1;
if (state === 'queued') return action === 'rebuild' ? 5 : 0;
const stages = action === 'destroy' ? ['Starting', ...destroyStages]
: action === 'rebuild' ? ['Starting', ...destroyStages, 'Queued for server creation', ...createStages]
: ['Starting', ...createStages];
const index = stages.findIndex(value => stage === value || stage.startsWith(value + ':'));
if (index < 0 && state === 'awaiting-health') return action === 'rebuild' ? 9 : 4;
return Math.max(0, index);
let now = $state(Date.parse(initial.generatedAt));
let launchOpen = $state(false);
const controller = createProvisioningRefresh<ProvisioningDetail>({ validate: isProvisioningDetail, onChange: value => { view = value; } });
const detail = $derived(view.detail);
const deployment = $derived(detail?.deployment);
const outcome = $derived(detail ? provisioningOutcome(detail) : null);
const returnTo = $derived.by(() => {
const requested = $page.url.searchParams.get('returnTo');
if (requested) return provisioningReturn(requested);
if (detail?.collection) return { href: `/collections/launches/${encodeURIComponent(detail.collection.id)}`, label: `Back to ${detail.collection.name}` };
if (detail?.sources.length === 1 && detail.sources[0].href) return { href: detail.sources[0].href, label: `Back to ${detail.sources[0].label}` };
return provisioningReturn(null);
});
let explanation = $derived.by(() => {
if (!deployment) return '';
if (deployment.server_deleted_at) return 'This server was deleted. Its recorded operation and results are preserved below.';
if (detail?.canResumeRebuild) return 'The old Komodo instance was deleted. Resume file cleanup, then create a fresh instance in the reserved server slot.';
if (['start', 'restart', 'stop'].includes(deployment.action)) {
if (deployment.state === 'done') return deployment.action === 'stop' ? 'Komodo confirmed the container is stopped. Start it again when ready.' : 'The container is healthy and RCON responds.';
if (['failed', 'interrupted'].includes(deployment.state)) return 'Inspect the server before submitting another operation. An uncertain response may have reached Komodo.';
return deployment.state === 'waiting' ? 'Waiting for other work and a safe server state before continuing.' : 'The request is saved. Verifying the resulting container state automatically.';
}
if (deployment.state === 'cancelled') return 'The queued removal was cancelled before removal began. The server and its files were kept.';
if (detail?.canResumeCleanup) return 'Komodo deletion succeeded. Resume the remaining file and database cleanup; the removed container does not need to answer RCON.';
if (deployment.state === 'waiting' && deployment.stage === 'Removing server files and registry') return 'Local cleanup is queued. Komodo deletion has already succeeded.';
if (deployment.state === 'done') return ['create', 'rebuild'].includes(deployment.action)
? 'The container is healthy, RCON responds, and the reported engine map matches your selection.'
: 'The container, deployment, and server registration have been removed.';
if (deployment.state === 'warning') return 'The operation finished with the warnings below. Review any remaining cleanup.';
if (deployment.state === 'failed' || deployment.state === 'interrupted') return detail?.canResumeVerification
? 'The server is still registered. Inspect its status, then resume health verification without creating or restarting a container.'
: 'Inspect the error and any remaining container or files before launching again. The activity below records how far this operation reached.';
if (deployment.state === 'queued') return 'Your request is saved in the launch queue. A startup slot opens after an earlier server passes its container, RCON, and map checks.';
if (deployment.action === 'rebuild' && deployment.state === 'waiting') return 'The rebuild waits for other work to finish, an empty server, and available recovery and startup capacity. The old instance and all of its files will be deleted before a fresh instance is created.';
if (deployment.state === 'waiting') return 'The server stays online while players finish. Removal begins only after a live check confirms it is empty.';
if (deployment.stage.startsWith('Waiting for selected map')) return 'Container and RCON checks passed. Waiting for RCON to report the selected engine map. Inspect the server configuration if this persists.';
if (deployment.stage.startsWith('Waiting for RCON')) return 'The container is healthy. Waiting for the game to answer RCON; check its startup logs, password, and network if this persists.';
if (deployment.state === 'awaiting-health') return 'Waiting for the container to become healthy. Verification runs automatically; startup logs can help explain a delay.';
return ['create', 'rebuild'].includes(deployment.action) ? 'Installing the latest promoted setup and starting its container. Health checks follow automatically.' : 'Removing the empty server and recording cleanup results.';
const serverHref = $derived(deployment && deployment.server_exists && !deployment.server_deleted_at ? `/servers/${encodeURIComponent(deployment.server)}` : null);
const recipeStatus = $derived(detail?.progress?.recipeStatus ?? null);
const recipeHeading = $derived(recipeStatus === 'queued' ? 'Setup captured when queued' : recipeStatus === 'attempted' ? 'Files selected for this attempt' : 'Recorded launch files');
$effect(() => {
const snapshot = data.detail;
untrack(() => {
launchOpen = false;
now = Date.parse(snapshot.generatedAt);
controller.setOperation(snapshot.deployment.id, snapshot, snapshot.generatedAt);
});
});
function duration(start: string, end: number) {
const seconds = Math.max(0, Math.floor((end - new Date(start).getTime()) / 1000));
if (seconds < 60) return `${seconds}s`;
if (seconds < 3600) return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
return `${Math.floor(seconds / 3600)}h ${Math.floor(seconds % 3600 / 60)}m`;
}
let stageStarted = $derived(detail?.events.slice().reverse().find(event => event.message === deployment?.stage)?.created_at ?? deployment?.started_at);
async function resume() {
if (!deployment || resuming) return;
resuming = true;
onMount(() => pollVisible(() => { now = Date.now(); return controller.refresh(); }, 5000, false));
onDestroy(() => controller.dispose());
function fmt(value: string | null | undefined) { return value ? $timeFormat.dateTime(value) : 'Not recorded'; }
async function copyLink() {
try {
const res = await fetch(`/api/provisioning/${encodeURIComponent(deployment.id)}/resume`, { method: 'POST' });
const body = await res.json();
if (!res.ok) throw new Error(body.error ?? 'Could not resume the operation.');
await load();
} catch (err) { error = (err as Error).message; }
finally { resuming = false; }
const url = new URL($page.url);
url.search = ''; url.hash = '';
await navigator.clipboard.writeText(url.toString());
toast('Operation link copied.', { variant: 'success' });
} catch { toast('Copy unavailable. Copy this page’s address from your browser.', { variant: 'warning' }); }
}
async function cancelRemoval() {
if (!deployment || resuming) return;
resuming = true; error = '';
try {
const response = await fetch(`/api/provisioning/${encodeURIComponent(deployment.id)}/cancel`, { method: 'POST' });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not cancel this request.');
await load();
} catch (err) { error = (err as Error).message; await load(); error = (err as Error).message; }
finally { resuming = false; }
}
async function load() {
if (refreshing) return;
refreshing = true;
const id = $page.params.id ?? '';
try {
const res = await fetch('/api/provisioning/' + encodeURIComponent(id));
if (!res.ok) throw new Error(res.status === 404 ? 'Provisioning operation not found.' : 'Could not refresh provisioning operation.');
const body = await res.json();
if ($page.params.id === id) { detail = body; error = ''; }
} catch (err) { error = (err as Error).message; }
finally { refreshing = false; }
}
$effect(() => { $page.params.id; detail = null; void load(); });
onMount(() => pollVisible(() => { now = Date.now(); return load(); }, 5000, false));
function fmt(iso: string) { return $timeFormat.dateTime(iso); }
</script>
<svelte:head><title>{deployment ? actionLabels[deployment.action] + ' ' + deployment.server : 'Provisioning'} | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title={deployment ? actionLabels[deployment.action] + ' server' : 'Provisioning'} subtitle={deployment ? serverTitle(deployment.server) : undefined}>
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Provisioning', href: '/provisioning' }, { label: deployment ? serverTitle(deployment.server) : 'Details' }]} />{/snippet}
</SectionHeader>
{#if error}<p role="alert" class="error">{error} <button onclick={load}>Retry</button></p>{/if}
{#if detail?.collection}<a href="/collections/launches/{detail.collection.id}">← Back to {detail.collection.name} collection</a>{/if}
{#if deployment}
<section class="summary">
<div class="top"><StatusBadge state={deployment.state} />
{#if deployment.server_deleted_at}<Badge>Deleted</Badge>{:else if deployment.server_exists}<a href="/servers/{encodeURIComponent(deployment.server)}">View server</a>{/if}
</div>
<h2>{deployment.state === 'done' ? actionCompleted[deployment.action] : deployment.stage}</h2>
<p class="explanation" aria-live="polite">{explanation}</p>
<div class="timing"><span>Total time <strong>{duration(deployment.started_at, deployment.finished_at ? new Date(deployment.finished_at).getTime() : now)}</strong></span>{#if active && stageStarted}<span>This step <strong>{duration(stageStarted, now)}</strong></span>{/if}</div>
{#if detail?.canResumeVerification && !deployment.server_deleted_at}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume health verification</Button></div>{/if}
{#if detail?.canResumeRebuild && !deployment.server_deleted_at}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume rebuild cleanup</Button></div>{/if}
{#if detail?.canResumeCleanup && !deployment.server_deleted_at}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume cleanup</Button></div>{/if}
{#if detail?.canCancelRemoval && !deployment.server_deleted_at}<div><Button disabled={resuming} loading={resuming} onclick={cancelRemoval}>Cancel queued removal</Button><p class="muted">Available until removal begins. Cancelling keeps the server and its files.</p></div>{/if}
{#if deployment.server_exists && !deployment.server_deleted_at && deployment.state !== 'done' && deployment.state !== 'cancelled'}<div class="recovery-links"><a href="/servers/{encodeURIComponent(deployment.server)}?tab=logs">Open startup logs</a><a href="/servers/{encodeURIComponent(deployment.server)}?tab=diagnostics">Diagnose connection</a>{#if deployment.stage.includes('map')}<a href="/servers/{encodeURIComponent(deployment.server)}?tab=config">Review map configuration</a>{/if}</div>{/if}
{#if deployment.error}<p role="alert" class="error">{deployment.error}</p>{/if}
{#each warnings as warning}<p class="warning">{warning}</p>{/each}
{#if deployment.expected_map}<p>Expected map: <strong>{deployment.expected_map}</strong></p>{/if}
<dl><div><dt>Started</dt><dd>{fmt(deployment.started_at)}</dd></div><div><dt>Finished</dt><dd>{deployment.finished_at ? fmt(deployment.finished_at) : 'In progress'}</dd></div><div><dt>Requested by</dt><dd>{deployment.triggered_by}</dd></div></dl>
{#if detail?.sources.length}<div class="recovery-links" aria-label="Requesting work">{#each detail.sources as source}<a href={source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}` : source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}` : source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}` : `/maintenance?view=activity#job-${source.source_id}`}>{source.kind === 'campaign' ? 'Campaign' : source.kind === 'beta' ? 'Beta' : source.kind === 'watchdog' ? 'Maintenance finding' : 'Maintenance'} {source.source_id}</a>{/each}</div>{/if}
</section>
{#if detail?.recipe}<section class="saved-recipe"><div class="top"><h2>{detail.blueprintName ? `Launched from ${detail.blueprintName}` : 'Recorded launch files'}</h2><Button variant="ghost" onclick={() => open = true}>Launch another server →</Button></div><p class="muted">Launch this map with the latest promoted files using a fresh server ID and port. This does not repair the existing server.</p><LaunchRecipe recipe={detail.recipe} historical /></section><DeployServerDialog bind:open sourceDeploymentId={deployment.id} />{/if}
<OperationHistory steps={steps.map((label,index): OperationStep => ({label,state:deployment.state === 'cancelled' ? 'pending' : index < currentStep ? 'done' : index === currentStep ? (!active && ['failed','interrupted'].includes(deployment.state) ? 'failed' : 'current') : 'pending'}))} events={detail?.events ?? []} {active} />
<svelte:head><title>{deployment ? `${actionLabels[deployment.action]} ${deployment.server}` : 'Provisioning'} | Electric Cherry</title></svelte:head>
{:else if !error}<p>Loading provisioning operation...</p>{/if}
<div class="page">
<SectionHeader title={deployment ? `${actionLabels[deployment.action]} · ${serverTitle(deployment.server)}` : 'Provisioning operation'} subtitle={deployment?.server}>
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Provisioning', href: returnTo.href.startsWith('/provisioning') ? returnTo.href : '/provisioning' }, { label: 'Operation' }]} />{/snippet}
{#snippet actions()}{#if serverHref}<Button href={serverHref}>View server</Button>{/if}{/snippet}
</SectionHeader>
<div class="identity-row">
{#if deployment}<div class="operation-id"><span>Operation</span><CopyValue value={deployment.id} label="operation ID" /></div>{/if}
<div class="identity-links"><a href={returnTo.href}>← {returnTo.label}</a><Button size="sm" variant="ghost" onclick={copyLink}>Copy link</Button></div>
</div>
{#if view.loadError}<div class="refresh-error" role="alert"><div><strong>Refresh unavailable</strong><p>{view.loadError} {detail ? 'Showing the last successfully loaded record.' : ''}</p></div><Button size="sm" disabled={view.refreshing || !!view.busy} onclick={() => controller.forceRefresh()}>Retry refresh</Button></div>{/if}
{#if deployment && detail && outcome}
<section class="outcome" class:attention={['failed', 'interrupted', 'warning'].includes(deployment.state)} aria-labelledby="outcome-title">
<div class="outcome-top"><div class="badges"><StatusBadge state={deployment.state} />{#if deployment.server_deleted_at}<Badge>Deleted identity</Badge>{/if}{#if detail.progress?.recovery?.state === 'held'}<Badge variant="warning">Recovery capacity held</Badge>{/if}</div><span class="record-label">Operation outcome</span></div>
<div class="outcome-message" aria-live="polite" aria-atomic="true"><h2 id="outcome-title">{outcome.headline}</h2><p>{outcome.description}</p></div>
{#if deployment.error}<div class="failure" role="alert"><strong>Recorded error</strong><p>{deployment.error}</p></div>{/if}
{#if detail.activity.failureExcerpt && detail.activity.failureExcerpt.message !== deployment.error && ['failed', 'interrupted'].includes(deployment.state)}<p class="failure-excerpt"><span>Last failure event · {fmt(detail.activity.failureExcerpt.created_at)}</span>{detail.activity.failureExcerpt.message}</p>{/if}
{#if detail.warningMessages.length}<div class="warnings"><h3>Recorded warnings</h3><ul>{#each detail.warningMessages as warning}<li>{warning}</li>{/each}</ul></div>{/if}
{#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancelRemoval}
<div class="recovery-actions">
{#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. The existing instance will be observed without another creation or restart.')}>Resume health verification</Button>{/if}
{#if detail.canResumeRebuild}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Rebuild cleanup requested. Fresh creation waits for local cleanup to finish.')}>Resume rebuild cleanup</Button>{/if}
{#if detail.canResumeCleanup}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Local cleanup requested. Remote container deletion will not be repeated.')}>Resume cleanup</Button>{/if}
{#if detail.canCancelRemoval}<div><Button disabled={!!view.busy} loading={view.busy === 'cancel'} onclick={() => controller.act('cancel', 'Queued removal cancelled. The server and its files were kept.')}>Cancel queued removal</Button><p class="action-help">Available until removal begins. Cancelling keeps the server and its files.</p></div>{/if}
</div>
{/if}
{#if view.actionError}<div class="action-error" role="alert"><strong>Request not confirmed</strong><p>{view.actionError}</p></div>{/if}
{#if view.notice}<p class="notice" role="status">{view.notice}</p>{/if}
{#if detail.progress?.blocker?.href}<a class="dependency" href={detail.progress.blocker.href}>{detail.progress.blocker.label ?? 'View blocking work'} →</a>{/if}
{#if detail.progress?.recovery}<div class="recovery-note"><strong>{detail.progress.recovery.state === 'held' ? 'Recovery remains in progress' : detail.progress.recovery.state === 'released' ? 'Recovery capacity released' : detail.progress.recovery.state === 'transferred' ? 'Recovery transferred' : 'Recovery state not confirmed'}</strong><p>{detail.progress.recovery.message}</p><span>Recorded {fmt(detail.progress.recovery.at)}</span>{#if detail.progress.recovery.operationId && detail.progress.recovery.operationId !== deployment.id}<a href={`/provisioning/${encodeURIComponent(detail.progress.recovery.operationId)}`}>View recovery operation →</a>{/if}</div>{/if}
{#if serverHref && !['done', 'cancelled'].includes(deployment.state)}<div class="diagnostics"><a href={`${serverHref}?tab=logs`}>Open server logs</a><a href={`${serverHref}?tab=diagnostics`}>Diagnose connection</a>{#if detail.progress?.phase === 'verify-map' || detail.progress?.checks.some(check => check.key === 'map' && check.result !== 'pass')}<a href={`${serverHref}?tab=config`}>Review map configuration</a>{/if}</div>{/if}
<div class="timing">
<div><span>{outcome.active ? 'Total elapsed' : 'Recorded duration'}</span><strong>{elapsedTime(deployment.started_at, deployment.finished_at ? Date.parse(deployment.finished_at) : now)}</strong></div>
{#if outcome.active}<div><span>Time in phase</span><strong>{elapsedTime(detail.progress?.phaseStartedAt, now)}</strong></div>{/if}
<div><span>Last evaluated</span><time datetime={detail.progress?.observedAt ?? undefined}>{fmt(detail.progress?.observedAt)}</time></div>
</div>
<div class="freshness"><span>Page updated <time datetime={view.refreshedAt ?? undefined}>{fmt(view.refreshedAt)}</time> · updates automatically</span><Button size="sm" variant="ghost" disabled={view.refreshing || !!view.busy} loading={view.refreshing} onclick={() => controller.forceRefresh()}>Refresh</Button></div>
</section>
{#key deployment.id}
<div class="detail-grid">
<div class="execution-column"><ProvisioningProgress {detail} /><ProvisioningActivity {detail} /></div>
<aside class="context-column" aria-label="Operation context">
<Card as="section" padding="md" ariaLabel="Operation details">
<div class="context-section"><h2>Operation details</h2><dl class="metadata">
<div><dt>Requested by</dt><dd>{deployment.triggered_by}</dd></div>
<div><dt>Requested at</dt><dd><time datetime={deployment.started_at}>{fmt(deployment.started_at)}</time></dd></div>
<div><dt>Submission attempted</dt><dd>{fmt(deployment.submitted_at)}</dd></div>
<div><dt>Finished</dt><dd>{deployment.finished_at ? fmt(deployment.finished_at) : outcome.active ? 'In progress' : 'Not recorded'}</dd></div>
{#if deployment.server_deleted_at}<div><dt>Identity deleted</dt><dd>{fmt(deployment.server_deleted_at)}</dd></div>{/if}
</dl></div>
</Card>
{#if detail.sources.length || detail.collection}<Card as="section" padding="md" ariaLabel="Related work"><div class="context-section"><h2>Related work</h2><p>The requesting work may have further checks after this operation finishes.</p><ul class="related">
{#if detail.collection}<li><a href={`/collections/launches/${encodeURIComponent(detail.collection.id)}`}>{detail.collection.name}</a><span>Collection launch</span></li>{/if}
{#each detail.sources as source (`${source.kind}:${source.source_id}`)}<li>{#if source.href}<a href={source.href}>{source.label}</a>{:else}<strong>{source.label}</strong><span>Source record no longer available</span>{/if}<code>{source.source_id}</code></li>{/each}
</ul></div></Card>{/if}
{#if detail.capacity.pools.length}<Card as="section" padding="md" ariaLabel="Capacity"><div class="context-section"><h2>Capacity</h2>
{#each detail.capacity.pools as pool (pool.kind)}<div class="capacity-pool"><div class="capacity-heading"><strong>{pool.kind === 'startup' ? 'Startup slots' : 'Restart recovery slots'}</strong><span>{pool.used} / {pool.limit} in use</span></div>
{#if pool.ownsSlot}<p>This operation holds a slot.</p>{/if}
{#if pool.requestOrder !== null}<p>Request order: {pool.requestOrder}. Eligible requests may pass blocked work; this is not a guaranteed start order.</p>{/if}
{#if pool.holders.length}<details><summary>Current slot holders</summary><ul>{#each pool.holders as holder}<li>{#if holder.id}<a href={`/provisioning/${encodeURIComponent(holder.id)}`}>{serverTitle(holder.server)}</a>{:else}<span>{serverTitle(holder.server)} · operation not recorded</span>{/if}</li>{/each}</ul></details>{/if}
</div>{/each}<p class="small">Capacity snapshot {fmt(detail.capacity.evaluatedAt)}</p>
</div></Card>{/if}
<Card as="section" padding="md" ariaLabel="Recorded launch inputs"><div class="context-section recipe"><h2>{recipeHeading}</h2>
{#if detail.recipe}
{#if detail.blueprintName}<p>From {detail.blueprintName}</p>{/if}
<p>{recipeStatus === 'queued' ? 'This capture does not pin versions. The latest promoted setup is selected again when creation begins.' : recipeStatus === 'attempted' ? 'These files were selected when creation began. Selection alone does not confirm successful installation.' : 'This is the saved recipe for this operation. The point at which it was selected was not recorded.'}</p>
<p class="small">Captured <time datetime={detail.recipe.capturedAt}>{fmt(detail.recipe.capturedAt)}</time></p>
<LaunchRecipe recipe={detail.recipe} historical selectionStatus={recipeStatus} />
<div class="launch-another"><Button variant="ghost" onclick={() => launchOpen = true}>Launch another server →</Button><p>Creates a separate server with a fresh ID and port using the latest promoted files. It does not repair this instance.</p></div>
{:else}<p>{['create', 'rebuild'].includes(deployment.action) ? 'No launch recipe was recorded for this operation.' : 'This operation uses the existing instance. No new launch files were selected.'}</p>{/if}
</div></Card>
</aside>
</div>
{#if detail.recipe}<DeployServerDialog bind:open={launchOpen} sourceDeploymentId={deployment.id} />{/if}
{/key}
{:else}<div class="loading" role="status">Loading provisioning operation…</div>{/if}
</div>
<style>
.recovery-links { display: flex; gap: 16px; flex-wrap: wrap; font-size: 14px; }
.page { display: grid; gap: 28px; } section { display: grid; gap: 16px; }
.summary { padding: 24px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); }
.saved-recipe { padding: 24px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface-sunken); }.explanation { color: var(--text-secondary); max-width: 800px; }.timing { display: flex; gap: 24px; font-size: 12px; color: var(--text-secondary); }.timing strong { color: var(--text-primary); margin-left: 8px; font-family: var(--font-mono); font-weight: 500; }
.top { display: flex; align-items: center; justify-content: space-between; gap: 16px; flex-wrap: wrap; overflow-wrap: anywhere; }
h2 { font-size: 18px; margin: 0; } p { margin: 0; line-height: 1.6; } a { color: var(--accent-primary-hover); }
dl { display: flex; gap: 32px; flex-wrap: wrap; margin: 0; } dt, .muted { color: var(--text-secondary); font-size: 13px; } dd { margin: 6px 0 0; font-size: 14px; }
.error { color: var(--state-danger); } .warning { color: var(--state-warning); }
@media (max-width: 640px) {
.timing { flex-direction: column; gap: var(--space-2); }
}
.page { display: grid; gap: var(--space-6); min-width: 0; }
.page :global(.subtitle), .page :global(.title) { overflow-wrap: anywhere; }
.page :global(.btn) { white-space: normal; }
h2, h3, p { margin: 0; }a { color: var(--accent-primary-hover); text-underline-offset: 3px; }
a:focus-visible, summary:focus-visible { outline: 2px solid var(--border-focus); outline-offset: 3px; border-radius: 2px; }
p { line-height: 1.65; overflow-wrap: anywhere; }
.identity-row, .operation-id, .identity-links, .badges, .outcome-top, .diagnostics, .freshness, .recovery-actions { display: flex; align-items: center; gap: var(--space-3); flex-wrap: wrap; }
.identity-row { justify-content: space-between; margin-top: calc(-1 * var(--space-3)); gap: var(--space-2) var(--space-5); }.operation-id { gap: var(--space-2); min-width: 0; }.operation-id > span, .identity-links { font-size: var(--text-xs); color: var(--text-tertiary); }.identity-links { gap: var(--space-4); }
.outcome { display: grid; gap: var(--space-5); padding: var(--space-6); border: 1px solid var(--border-strong); border-radius: var(--radius-lg); background: var(--surface); }
.outcome.attention { border-color: color-mix(in srgb, var(--state-warning) 35%, var(--border-strong)); }.outcome-top { justify-content: space-between; }.record-label { color: var(--text-tertiary); font-size: var(--text-xs); }.outcome h2 { font-size: var(--text-xl); line-height: 1.35; letter-spacing: var(--tracking-tight); overflow-wrap: anywhere; }
.outcome-message { display: grid; gap: var(--space-2); }.outcome-message p { color: var(--text-secondary); font-size: var(--text-base); max-width: 85ch; }
.failure, .action-error, .refresh-error { border: 1px solid color-mix(in srgb, var(--state-danger) 35%, var(--border-subtle)); border-radius: var(--radius-md); padding: var(--space-4); color: var(--state-danger); font-size: var(--text-sm); }.failure p, .action-error p { margin-top: var(--space-2); color: var(--text-primary); }.refresh-error { display: flex; justify-content: space-between; align-items: center; gap: var(--space-4); flex-wrap: wrap; }.refresh-error p { color: var(--text-secondary); margin-top: var(--space-1); }
.failure-excerpt { display: grid; gap: var(--space-1); font-size: var(--text-sm); color: var(--text-secondary); }.failure-excerpt span { font-size: var(--text-xs); color: var(--text-tertiary); }
.warnings { border-left: 3px solid var(--state-warning); padding-left: var(--space-4); font-size: var(--text-sm); }.warnings h3 { font-size: var(--text-sm); color: var(--state-warning); }.warnings ul { margin: var(--space-2) 0 0; padding-left: var(--space-4); line-height: 1.6; }.warnings li { overflow-wrap: anywhere; }
.notice { color: var(--state-online); font-size: var(--text-sm); }.action-help { color: var(--text-secondary); font-size: var(--text-xs); margin-top: var(--space-2); }.dependency, .diagnostics { font-size: var(--text-sm); }.diagnostics { gap: var(--space-3) var(--space-5); }
.recovery-note { display: grid; gap: var(--space-2); padding: var(--space-4); border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface-sunken); font-size: var(--text-sm); }.recovery-note p, .recovery-note span { color: var(--text-secondary); }.recovery-note span { font-size: var(--text-xs); }
.timing { border-top: 1px solid var(--border-subtle); padding-top: var(--space-5); display: flex; flex-wrap: wrap; gap: var(--space-4) var(--space-8); }.timing > div { display: grid; gap: var(--space-2); }.timing span { font-size: var(--text-xs); color: var(--text-tertiary); }.timing strong { font-size: var(--text-md); font-variant-numeric: tabular-nums; font-weight: var(--weight-medium); }.timing time { font-size: var(--text-sm); color: var(--text-secondary); }
.freshness { justify-content: space-between; color: var(--text-tertiary); font-size: var(--text-xs); line-height: 1.65; margin-top: calc(-1 * var(--space-2)); }
.detail-grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(280px, 340px); gap: var(--space-8); align-items: start; }.execution-column { display: grid; gap: var(--space-8); min-width: 0; }.context-column { display: grid; gap: var(--space-5); min-width: 0; }.context-section { display: grid; gap: var(--space-4); min-width: 0; }.context-section h2 { font-size: var(--text-md); }.context-section p { font-size: var(--text-sm); color: var(--text-secondary); }.context-section p.small { font-size: var(--text-xs); }
.metadata { display: grid; gap: var(--space-4); margin: 0; }.metadata dt { color: var(--text-tertiary); font-size: var(--text-xs); }.metadata dd { margin: var(--space-1) 0 0; font-size: var(--text-sm); line-height: 1.6; overflow-wrap: anywhere; }
.related, .capacity-pool ul { list-style: none; padding: 0; margin: 0; display: grid; gap: var(--space-4); }.related li { display: grid; gap: var(--space-1); font-size: var(--text-sm); overflow-wrap: anywhere; }.related span, .related code { color: var(--text-tertiary); font-size: var(--text-xs); }.related strong { font-weight: var(--weight-medium); }code { font-family: var(--font-mono); }
.capacity-pool { display: grid; gap: var(--space-3); font-size: var(--text-xs); }.capacity-pool + .capacity-pool { border-top: 1px solid var(--border-subtle); padding-top: var(--space-4); }.capacity-heading { display: flex; justify-content: space-between; gap: var(--space-2); flex-wrap: wrap; }.capacity-heading strong { font-weight: var(--weight-medium); }.capacity-heading span { color: var(--text-secondary); }.capacity-pool summary { cursor: pointer; color: var(--text-secondary); }.capacity-pool ul { gap: var(--space-2); margin-top: var(--space-3); line-height: 1.6; }
.launch-another { display: grid; gap: var(--space-2); justify-items: start; border-top: 1px solid var(--border-subtle); padding-top: var(--space-4); }.launch-another p { font-size: var(--text-xs); }
.loading { min-height: 240px; padding: var(--space-6); border: 1px solid var(--border-subtle); border-radius: var(--radius-lg); color: var(--text-secondary); }
@media (max-width: 1000px) { .detail-grid { grid-template-columns: minmax(0, 1fr); }.context-column { grid-template-columns: repeat(2, minmax(0, 1fr)); } }
@media (max-width: 600px) { .page { gap: var(--space-5); }.outcome { padding: var(--space-4); gap: var(--space-4); }.outcome h2 { font-size: var(--text-lg); }.outcome-message p { font-size: var(--text-sm); }.context-column { grid-template-columns: minmax(0, 1fr); }.identity-row, .identity-links { align-items: flex-start; }.operation-id { display: block; }.operation-id > span { display: block; margin-bottom: var(--space-1); }.outcome-top { align-items: flex-start; }.record-label { display: none; }.timing { gap: var(--space-4); }.timing > div:last-child { flex-basis: 100%; }.freshness { align-items: flex-start; }.freshness > span { flex: 1 1 170px; }.detail-grid { gap: var(--space-6); } }
</style>