josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Watchdog recovery and drift rebuild workflows

Browse Source
This commit is contained in:
josh
2026-09-27 15:13:38 -04:00
parent a7044f21a1
commit b9f5902945
36 changed files with 2131 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/requests.ts
+118 -4
View File
@@ -1,14 +1,16 @@
import { db } from '../db/index.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { getServer } from '../fleet/servers.js';
import { beginDeployment, deploymentStage, recordDeploymentEvent } from './history.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
import { queueRebuildServer } from './instances.js';
import { restartRequestInFlight } from './restarts.js';
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
export interface LifecycleOperation {
id: string; server: string; action: string; state: string; stage: string;
submitted_at: string | null; finished_at: string | null; checked_at: string | null;
players: number | null; error: string | null;
started_at: string;
}
export interface RestartSource { kind: 'campaign' | 'beta' | 'maintenance'; id: string }
export function provisioningOperation(id: string): LifecycleOperation | undefined {
@@ -115,19 +117,131 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
return db().transaction(() => {
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
if (existing) return existing.action === action
? { ok: true as const, deploymentId: existing.id }
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
if (existing) {
if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
return { ok: true as const, deploymentId: existing.id };
}
const row = getServer(server);
if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
if (action === 'rebuild') {
const result = queueRebuildServer(server);
if (result.ok) db().prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?").run(server);
return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
: { ok: false as const, status: 400, error: result.message };
}
const id = beginDeployment(server, action);
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
return { ok: true as const, deploymentId: id };
})();
}
type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string };
function watchdogRequestGuard(server: string, incidentId: string): Extract<LifecycleRequestResult, { ok: false }> | null {
const incident = db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(incidentId, server);
if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' };
if (db().prepare("SELECT 1 FROM settings WHERE key = 'watchdog_enabled' AND value = '0'").get()) {
return { ok: false, status: 409, error: 'Watchdog is paused.' };
}
const row = getServer(server);
if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' };
if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' };
if (!komodoConfigured()) return { ok: false, status: 503, error: 'Configure Komodo before provisioning.' };
return null;
}
function existingWatchdogAction(incidentId: string, action: 'restart' | 'rebuild'): string | undefined {
return (db().prepare(`SELECT a.operation_id FROM watchdog_actions a JOIN watchdog_incidents i ON i.id = a.incident_id
WHERE a.incident_id = ? AND a.action = ? AND a.cycle = i.cycle`).get(incidentId, action) as { operation_id: string } | undefined)?.operation_id;
}
function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult {
let id: string;
if (action === 'rebuild') {
const result = queueRebuildServer(server, `watchdog:${incidentId}`);
if (!result.ok) return { ok: false, status: 400, error: result.message };
id = result.deploymentId;
} else {
id = beginDeployment(server, action, `watchdog:${incidentId}`);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty');
}
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server);
return { ok: true, deploymentId: id };
}
/** Watchdog submits durable work; Provisioning remains the only execution owner. */
export function requestWatchdogLifecycle(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult {
return db().transaction(() => {
const previous = existingWatchdogAction(incidentId, action);
if (previous) return { ok: true as const, deploymentId: previous };
const error = watchdogRequestGuard(server, incidentId);
if (error) return error;
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
WHERE d.server = ? AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`)
.get(server) as { id: string; action: string; source_id: string | null } | undefined;
if (pending) return pending.source_id === incidentId && pending.action === action
? { ok: true as const, deploymentId: pending.id }
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
const next = queueWatchdogAction(server, action, incidentId);
if (!next.ok) return next;
// An operator retry starts a new incident cycle. Keep the previous
// failed attempt's capacity, but let that new attempt use it instead
// of waiting forever behind its own incident's terminal operation.
const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r
JOIN server_deployments d ON d.id = r.operation_id
JOIN watchdog_actions a ON a.operation_id = d.id AND a.incident_id = ?
JOIN watchdog_incidents i ON i.id = a.incident_id
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
if (retained && !restartRequestInFlight(server)) {
db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(next.deploymentId, server, retained.operation_id);
recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId);
recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id);
}
return next;
})();
}
/** Atomically reserve a replacement before handing it the failed restart's capacity. */
export function escalateWatchdogRestart(incidentId: string, operationId: string, recoveryTimeoutMs = 300_000): LifecycleRequestResult {
return db().transaction(() => {
const previous = existingWatchdogAction(incidentId, 'rebuild');
if (previous) return { ok: true as const, deploymentId: previous };
const operation = provisioningOperation(operationId);
if (!operation || operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state)
|| !db().prepare("SELECT 1 FROM provisioning_sources WHERE operation_id = ? AND kind = 'watchdog' AND source_id = ?").get(operationId, incidentId)) {
return { ok: false as const, status: 409, error: 'The restart is not eligible for Watchdog escalation.' };
}
const error = watchdogRequestGuard(operation.server, incidentId);
if (error) return error;
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
WHERE d.server = ? AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`)
.get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined;
if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId
? { ok: true as const, deploymentId: pending.id }
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as
{ operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined;
const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at);
if (restartRequestInFlight(operation.server) || (slot && (slot.operation_id !== operationId || slot.healthy_at))
|| !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) {
return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' };
}
const replacement = queueWatchdogAction(operation.server, 'rebuild', incidentId);
if (!replacement.ok) return replacement;
const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId;
if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason });
else recordDeploymentEvent(operationId, reason);
if (slot) db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(replacement.deploymentId, reason, operation.server, operationId);
recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server');
return replacement;
})();
}