josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve deleted server history

Browse Source 
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.
This commit is contained in:
josh
2026-09-27 21:48:55 -04:00
parent 53b29f726c
commit 981fb27c1c
61 changed files with 929 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/history.ts
+15 -9
View File
@@ -24,7 +24,7 @@ export function recordDeploymentEvent(id: string, message: string): void {
}
export function getDeployment(id: string) {
const deployment = db().prepare(`SELECT d.*, j.expected_map, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
const deployment = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id);
if (!deployment) return null;
const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
@@ -42,14 +42,14 @@ export function getDeployment(id: string) {
export const REMOVING_SERVER = 'Removing server files and registry';
export function canCancelRemoval(id: string): boolean {
return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
}
/** Competes atomically with the worker's claim, before any external removal. */
export function cancelRemoval(id: string): boolean {
return db().transaction(() => {
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER);
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER);
if (!result.changes) return false;
recordDeploymentEvent(id, 'Queued removal cancelled. The server and its files were kept.');
return true;
@@ -57,12 +57,18 @@ export function cancelRemoval(id: string): boolean {
}
export function claimRemoval(id: string): boolean {
return db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND action = 'destroy' AND state = 'waiting'").run(id).changes > 0;
return db().prepare(`UPDATE server_deployments SET state = 'running'
WHERE id = ? AND action = 'destroy' AND state = 'waiting'
AND (server_deleted_at IS NULL OR (stage = ?
AND NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = server_deployments.server)
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = server_deployments.server
AND newer.rowid > server_deployments.rowid)))`).run(id, REMOVING_SERVER).changes > 0;
}
export function canResumeCleanup(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d
WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server))
AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id, REMOVING_SERVER);
}
@@ -80,7 +86,7 @@ export function canResumeVerification(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d
JOIN server_creation_jobs j ON j.deployment_id = d.id
JOIN servers s ON s.name = j.server AND s.game_port = j.port
WHERE d.id = ? AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id);
@@ -88,7 +94,7 @@ export function canResumeVerification(id: string): boolean {
export function canResumeRebuild(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
WHERE d.id = ? AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id, REMOVING_SERVER);
}
@@ -120,8 +126,8 @@ export function finishDeployment(id: string, result: { ok: boolean; message?: st
}
export function listDeployments() {
return db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists,
(d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
return db().prepare(`SELECT d.*, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists,
(d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
FROM server_deployments d ORDER BY started_at DESC, rowid DESC`).all(REMOVING_SERVER);
}
src/lib/server/provisioning/instances.ts
+21 -31
View File
@@ -63,7 +63,7 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = []
const rows = db()
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
.all(`${baseName}_%`) as Array<{ name: string }>;
const taken = new Set<number>();
const prefix = `${baseName}_`;
@@ -87,7 +87,7 @@ export function nextAvailablePort(additionalPorts: number[] = [], releasingServe
const rows = db()
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
.all() as Array<{ name: string; game_port: number }>;
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
@@ -257,7 +257,7 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ
/** Caller has just verified emptiness and excluded conflicting work. */
export async function rebuildServer(id: string): Promise<void> {
const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
if (!job) return;
if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) {
// The cleanup checkpoint must survive capacity waits and app restarts.
@@ -487,17 +487,17 @@ const destroying = new Set<string>();
// Persist the request before returning so navigation and client disconnects cannot lose it.
function hasActiveCreation(name: string): boolean {
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
}
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action IN ('start','restart','stop','rebuild')
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild')
AND state IN ('waiting','queued','running')`).get(name)) {
return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
}
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
const operation = db().transaction(() => {
@@ -517,13 +517,18 @@ export function processDestructions(): Promise<void> {
}
async function runDestructions(): Promise<void> {
const queued = db().prepare("SELECT id, server, stage FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string; stage: string }[];
const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting'
AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at`).all(REMOVING_SERVER) as { id: string; server: string; stage: string; server_deleted_at: string | null }[];
for (const operation of queued) {
if (creating || destroying.has(operation.server)) continue;
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
destroying.add(operation.server);
try {
if (operation.stage === REMOVING_SERVER) {
if (operation.server_deleted_at && getServer(operation.server)) {
finishDeployment(operation.id, { ok: false, message: 'A replacement server is registered; local cleanup cannot safely resume.' });
continue;
}
const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
if (newer) {
@@ -619,25 +624,10 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
}
// Remove server-owned beta and deployment rows before the registry row.
// Test evidence cascades from runs; inventory and snapshots cascade from servers.
// The registry trigger preserves history and marks its old server identity.
// Live inventory, rollback snapshots, and restart slots still cascade away.
try {
const tx = db().transaction((n: string) => {
// The Watchdog's rebuild campaign observes this exact operation
// through the registry gap and retains its before/after file audit.
for (const table of ['deployment_servers', 'deployment_files']) {
db().prepare(`DELETE FROM ${table} WHERE server = ? AND deployment_id NOT IN (
SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL)`)
.run(n);
}
db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_coverage WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_file_changes WHERE server = ?').run(n);
db().prepare('DELETE FROM servers WHERE name = ?').run(n);
});
tx(name);
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
} catch (err) {
return {
ok: false,
@@ -706,7 +696,7 @@ export async function processCreations(): Promise<void> {
export function creationQueue() {
type Entry = { id: string; server: string; state: string; stage: string };
const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
ORDER BY started_at, rowid`).all() as Entry[];
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
}
@@ -726,7 +716,7 @@ async function runCreations(): Promise<CreationOutcomes> {
if (creating || destroying.size > 0) return outcomes;
// Restart recovery is observation-only: never repeat external creation or
// deployment. A known interrupted startup must retain its readiness gate.
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
for (const { id } of interrupted) {
if (resumeVerification(id)) continue;
// A queued removal does not make the interrupted container disappear.
@@ -748,14 +738,14 @@ async function runCreations(): Promise<CreationOutcomes> {
// Check only previously started servers. Newly launched servers keep their
// slot until a later pass, even if a health probe would answer immediately.
const checking = db().prepare(`SELECT j.*, COALESCE(d.submitted_at, d.started_at) AS submitted_at FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
WHERE d.server_deleted_at IS NULL AND d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
for (const job of checking) {
const id = job.deployment_id;
if (getServer(job.server)?.game_port !== job.port) {
finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
continue;
}
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
try {
const komodo = await komodoInspectHealthy(job.server);
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
@@ -787,7 +777,7 @@ async function runCreations(): Promise<CreationOutcomes> {
// API calls can cancel a selected job or resume an older startup while
// a previous provision awaits Komodo. Recheck without expanding this pass.
if (creationQueue().active.length >= fleetSettings().createConcurrency) break;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
const blocker = creationBlocker(id);
if (blocker) {
deploymentStage(id, blocker);
src/lib/server/provisioning/requests.ts
+9 -8
View File
@@ -28,7 +28,7 @@ const sourceTables = {
export function requestRestart(server: string, source: RestartSource): LifecycleOperation {
return db().transaction(() => {
const { table, key } = sourceTables[source.kind];
const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ?`)
const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`)
.get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined;
if (!row) throw new Error('Restart source no longer exists');
if (row.provisioning_id) {
@@ -45,14 +45,14 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
// The previous release already recorded some submissions in history.
// Bind to that exact interval, never to a newer restart of the server.
id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id
WHERE r.server = ? AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`)
WHERE r.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`)
.get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id;
if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id);
}
// Only not-yet-submitted campaign changes can share an activation. A later
// file deployment must never be credited to an already running restart.
if (source.kind === 'campaign' && !row.restarted_at) {
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.action = 'restart'
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart'
AND d.state = 'waiting' AND d.submitted_at IS NULL
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
@@ -107,9 +107,9 @@ export function cancelProvisioning(id: string, reason = 'Cancelled by requester'
/** Explicit caller retry starts a new job and keeps the old job as history. */
export function releaseRestartRequest(server: string, source: RestartSource): void {
const { table, key } = sourceTables[source.kind];
const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ?`).get(source.id, server) as { provisioning_id: string | null } | undefined;
const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).get(source.id, server) as { provisioning_id: string | null } | undefined;
if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt');
db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ?`).run(source.id, server);
db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).run(source.id, server);
}
export function requestLifecycle(server: string, action: RuntimeAction) {
@@ -117,6 +117,7 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
return db().transaction(() => {
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
AND server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
if (existing) {
if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
@@ -142,7 +143,7 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string };
function watchdogRequestGuard(server: string, incidentId: string): Extract<LifecycleRequestResult, { ok: false }> | null {
const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined;
const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined;
if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' };
if (!maintenanceAllowed(incident.kind)) {
return { ok: false, status: 409, error: 'This maintenance policy is paused.' };
@@ -183,7 +184,7 @@ export function requestWatchdogLifecycle(server: string, action: 'restart' | 're
if (error) return error;
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
WHERE d.server = ? AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`)
WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`)
.get(server) as { id: string; action: string; source_id: string | null } | undefined;
if (pending) return pending.source_id === incidentId && pending.action === action
? { ok: true as const, deploymentId: pending.id }
@@ -223,7 +224,7 @@ export function escalateWatchdogRestart(incidentId: string, operationId: string,
if (error) return error;
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
WHERE d.server = ? AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`)
WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`)
.get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined;
if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId
? { ok: true as const, deploymentId: pending.id }
src/lib/server/provisioning/restarts.ts
+16 -13
View File
@@ -7,7 +7,7 @@ import { deploymentStage, finishDeployment } from './history.js';
// Komodo queues restart work. Don't mistake the still-running old container
// for a completed restart when an API kick immediately follows the request.
const HEALTH_POLL_MS = 15_000;
const requestsInFlight = new Set<string>();
const requestsInFlight = new Map<string, string>();
export const restartRequestInFlight = (server: string) => requestsInFlight.has(server);
interface RestartSlot {
@@ -36,14 +36,14 @@ export function restartSettling(server: string): boolean {
}
export function rolloutWaitReason(server: string, operationId = ''): string | null {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
return 'Waiting for the current provisioning operation';
}
const slot = slotFor(server);
const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d
WHERE d.id = ? AND d.action IN ('restart','rebuild') AND EXISTS (
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS (
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId);
if (slot?.healthy_at === null && !ownHandoff) {
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
@@ -61,6 +61,7 @@ export type RolloutRestartResult =
export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
const operationId = options.operationId;
const waitReason = db().transaction(() => {
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
const reason = rolloutWaitReason(server, operationId);
if (reason) return reason;
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
@@ -73,35 +74,37 @@ export async function restartInRollout(server: string, options: { operationId: s
}).immediate();
if (waitReason) return { kind: 'queued', message: waitReason };
requestsInFlight.add(server);
requestsInFlight.set(server, operationId);
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
try {
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
if (result.ok) {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId);
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)');
return { kind: 'started' };
}
// These failures cannot have queued a remote restart. Other failures
// (especially a timeout) may have: retain capacity until recovery.
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server);
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
} else {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
}
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
return { kind: 'failed', message: `komodo: ${result.message}` };
} catch (err) {
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
const message = `komodo: ${(err as Error).message}`;
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
finishDeployment(operationId!, { ok: false, message });
return { kind: 'failed', message };
} finally {
// Start the grace period at completion, including slow/failed requests.
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?')
.run(new Date().toISOString(), server);
requestsInFlight.delete(server);
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ? AND operation_id = ?')
.run(new Date().toISOString(), server, operationId);
if (requestsInFlight.get(server) === operationId) requestsInFlight.delete(server);
}
}
@@ -121,7 +124,7 @@ async function refresh(): Promise<void> {
if (requestsInFlight.has(slot.server)) continue;
// Recovery capacity transferred to a rebuild remains reserved until
// teardown. Its health is verified by the creation worker, never here.
if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND (
if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND (
(d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health'))
OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`)
@@ -146,7 +149,7 @@ async function refresh(): Promise<void> {
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
if (!checked.changes) continue;
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) {
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
src/lib/server/provisioning/worker.ts
+10 -10
View File
@@ -23,7 +23,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
if (!sources.length) return 'ready';
// Watchdog intent gates every linked source, including its drift campaign.
for (const source of sources.filter(source => source.kind === 'watchdog')) {
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(source.source_id, operation.server)
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(source.source_id, operation.server)
|| getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
const incident = db().prepare('SELECT kind FROM watchdog_incidents WHERE id = ?').get(source.source_id) as { kind: 'health' | 'drift' };
if (!maintenanceAllowed(incident.kind)) return 'held';
@@ -31,21 +31,21 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
let held = false;
for (const source of sources) {
if (source.kind === 'campaign') {
const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ?')
const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL')
.get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined;
if (row?.provisioning_id !== operation.id || !row.files_changed) continue;
if (['pending','failed'].includes(row.restart_state)) return 'ready';
if (['held','manual'].includes(row.restart_state)) held = true;
} else if (source.kind === 'beta') {
const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ?`)
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`)
.get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined;
if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue;
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready';
if (row.outcome === 'paused') held = true;
} else if (source.kind === 'maintenance') {
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ?').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) {
if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) return 'held';
@@ -59,7 +59,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
function blocker(server: string, id: string): string | null {
const sources = sourcesFor(id);
const rank = priority(sources);
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ?
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[];
const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number };
if (others.some(other => other.state !== 'waiting' || other.action === 'destroy'
@@ -69,8 +69,8 @@ function blocker(server: string, id: string): string | null {
// Beta activation must not wait on the campaign rows that the beta itself
// holds. Campaign/maintenance callers likewise never block on their own job.
if (rank > 0 && db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ?
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
@@ -93,7 +93,7 @@ async function run(): Promise<void> {
await processDestructions();
await refreshRestartRollout();
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
ORDER BY started_at, rowid`).all() as Operation[];
for (const operation of operations) {
try { await processOperation(operation); }
@@ -140,9 +140,9 @@ async function processOperation(operation: Operation): Promise<void> {
}
// Requests can arrive during the probes. Recheck before claiming the action.
if (blocker(server, id) || sourceEligibility(operation) !== 'ready') return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
if (action === 'rebuild') { await rebuildServer(id); return; }
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id);
if (!claim.changes) return;
if (action === 'start' || action === 'restart') {
const result = await restartInRollout(server, { operationId: id, action });