josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Watchdog recovery and drift rebuild workflows

Browse Source
This commit is contained in:
josh
2026-09-27 15:13:38 -04:00
parent a7044f21a1
commit b9f5902945
36 changed files with 2131 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/index.ts
+1 -1
View File
@@ -1,6 +1,6 @@
// Request/observation boundary used by APIs, campaigns, beta tests, and schedules.
// Execution is driven exclusively by the provisioning worker.
export { requestLifecycle, requestRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
export type { LifecycleOperation, RestartSource } from './requests.js';
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextAvailablePort, nextLobbyNumber } from './instances.js';
export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';
src/lib/server/provisioning/instances.ts
+20 -13
View File
@@ -1,4 +1,4 @@
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
import { probePlayerCount, probeServerStatus } from '../fleet/status-evidence.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js';
@@ -229,7 +229,7 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
}
/** Reserve the same identity and port before removing any part of the instance. */
export function queueRebuildServer(name: string): CreateServerResult {
export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServerResult {
const server = getServer(name);
const game = gameOfServer(name);
const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
@@ -237,12 +237,12 @@ export function queueRebuildServer(name: string): CreateServerResult {
|| (server.host_dir && server.host_dir !== serverDir(name))) {
return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
}
if (!config.komodo.targetServer || !config.rcon.password) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
if (!config.komodo.targetServer || (!config.rcon.password && !triggeredBy.startsWith('watchdog:'))) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
// Validate that the replacement setup can be composed before destruction.
const recipe = captureRecipe(game, map.slug);
return db().transaction(() => {
const id = beginDeployment(name, 'rebuild');
const id = beginDeployment(name, 'rebuild', triggeredBy);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
@@ -436,6 +436,7 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
// Step 5: execute/Deploy — actually start the container.
deploymentStage(operation, 'Starting container');
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
const deployRes = await komodoDeploy(name);
if (!deployRes.ok) {
if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
@@ -541,7 +542,7 @@ async function runDestructions(): Promise<void> {
continue;
}
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
let players = await rconGetPlayerCount(operation.server);
let players = await probePlayerCount(operation.server);
if (players === null) {
const health = await komodoInspectHealthy(operation.server);
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
@@ -621,8 +622,13 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
// Test evidence cascades from runs; inventory and snapshots cascade from servers.
try {
const tx = db().transaction((n: string) => {
db().prepare('DELETE FROM deployment_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM deployment_files WHERE server = ?').run(n);
// The Watchdog's rebuild campaign observes this exact operation
// through the registry gap and retains its before/after file audit.
for (const table of ['deployment_servers', 'deployment_files']) {
db().prepare(`DELETE FROM ${table} WHERE server = ? AND deployment_id NOT IN (
SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL)`)
.run(n);
}
db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
@@ -647,6 +653,7 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
}
interface CreationJob {
deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
submitted_at?: string | null;
}
async function provisionCreation(id: string): Promise<CreateServerResult> {
@@ -739,7 +746,7 @@ async function runCreations(): Promise<CreationOutcomes> {
// Check only previously started servers. Newly launched servers keep their
// slot until a later pass, even if a health probe would answer immediately.
const checking = db().prepare(`SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
const checking = db().prepare(`SELECT j.*, COALESCE(d.submitted_at, d.started_at) AS submitted_at FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
for (const job of checking) {
const id = job.deployment_id;
@@ -751,14 +758,14 @@ async function runCreations(): Promise<CreationOutcomes> {
try {
const komodo = await komodoInspectHealthy(job.server);
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
const rcon = await rconStatusResult(job.server);
if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; }
const actualMap = rcon.status.map;
const status = await probeServerStatus(job.server, job.submitted_at);
if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason); continue; }
const actualMap = status.map;
if (actualMap !== job.expected_map) {
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map'));
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map'));
continue;
}
deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap);
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap);
finishDeployment(id, { ok: true });
} catch (err) {
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
src/lib/server/provisioning/requests.ts
+118 -4
View File
@@ -1,14 +1,16 @@
import { db } from '../db/index.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { getServer } from '../fleet/servers.js';
import { beginDeployment, deploymentStage, recordDeploymentEvent } from './history.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
import { queueRebuildServer } from './instances.js';
import { restartRequestInFlight } from './restarts.js';
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
export interface LifecycleOperation {
id: string; server: string; action: string; state: string; stage: string;
submitted_at: string | null; finished_at: string | null; checked_at: string | null;
players: number | null; error: string | null;
started_at: string;
}
export interface RestartSource { kind: 'campaign' | 'beta' | 'maintenance'; id: string }
export function provisioningOperation(id: string): LifecycleOperation | undefined {
@@ -115,19 +117,131 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
return db().transaction(() => {
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
if (existing) return existing.action === action
? { ok: true as const, deploymentId: existing.id }
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
if (existing) {
if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
return { ok: true as const, deploymentId: existing.id };
}
const row = getServer(server);
if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
if (action === 'rebuild') {
const result = queueRebuildServer(server);
if (result.ok) db().prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?").run(server);
return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
: { ok: false as const, status: 400, error: result.message };
}
const id = beginDeployment(server, action);
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
return { ok: true as const, deploymentId: id };
})();
}
type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string };
function watchdogRequestGuard(server: string, incidentId: string): Extract<LifecycleRequestResult, { ok: false }> | null {
const incident = db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(incidentId, server);
if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' };
if (db().prepare("SELECT 1 FROM settings WHERE key = 'watchdog_enabled' AND value = '0'").get()) {
return { ok: false, status: 409, error: 'Watchdog is paused.' };
}
const row = getServer(server);
if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' };
if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' };
if (!komodoConfigured()) return { ok: false, status: 503, error: 'Configure Komodo before provisioning.' };
return null;
}
function existingWatchdogAction(incidentId: string, action: 'restart' | 'rebuild'): string | undefined {
return (db().prepare(`SELECT a.operation_id FROM watchdog_actions a JOIN watchdog_incidents i ON i.id = a.incident_id
WHERE a.incident_id = ? AND a.action = ? AND a.cycle = i.cycle`).get(incidentId, action) as { operation_id: string } | undefined)?.operation_id;
}
function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult {
let id: string;
if (action === 'rebuild') {
const result = queueRebuildServer(server, `watchdog:${incidentId}`);
if (!result.ok) return { ok: false, status: 400, error: result.message };
id = result.deploymentId;
} else {
id = beginDeployment(server, action, `watchdog:${incidentId}`);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty');
}
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server);
return { ok: true, deploymentId: id };
}
/** Watchdog submits durable work; Provisioning remains the only execution owner. */
export function requestWatchdogLifecycle(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult {
return db().transaction(() => {
const previous = existingWatchdogAction(incidentId, action);
if (previous) return { ok: true as const, deploymentId: previous };
const error = watchdogRequestGuard(server, incidentId);
if (error) return error;
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
WHERE d.server = ? AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`)
.get(server) as { id: string; action: string; source_id: string | null } | undefined;
if (pending) return pending.source_id === incidentId && pending.action === action
? { ok: true as const, deploymentId: pending.id }
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
const next = queueWatchdogAction(server, action, incidentId);
if (!next.ok) return next;
// An operator retry starts a new incident cycle. Keep the previous
// failed attempt's capacity, but let that new attempt use it instead
// of waiting forever behind its own incident's terminal operation.
const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r
JOIN server_deployments d ON d.id = r.operation_id
JOIN watchdog_actions a ON a.operation_id = d.id AND a.incident_id = ?
JOIN watchdog_incidents i ON i.id = a.incident_id
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
if (retained && !restartRequestInFlight(server)) {
db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(next.deploymentId, server, retained.operation_id);
recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId);
recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id);
}
return next;
})();
}
/** Atomically reserve a replacement before handing it the failed restart's capacity. */
export function escalateWatchdogRestart(incidentId: string, operationId: string, recoveryTimeoutMs = 300_000): LifecycleRequestResult {
return db().transaction(() => {
const previous = existingWatchdogAction(incidentId, 'rebuild');
if (previous) return { ok: true as const, deploymentId: previous };
const operation = provisioningOperation(operationId);
if (!operation || operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state)
|| !db().prepare("SELECT 1 FROM provisioning_sources WHERE operation_id = ? AND kind = 'watchdog' AND source_id = ?").get(operationId, incidentId)) {
return { ok: false as const, status: 409, error: 'The restart is not eligible for Watchdog escalation.' };
}
const error = watchdogRequestGuard(operation.server, incidentId);
if (error) return error;
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
WHERE d.server = ? AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`)
.get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined;
if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId
? { ok: true as const, deploymentId: pending.id }
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as
{ operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined;
const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at);
if (restartRequestInFlight(operation.server) || (slot && (slot.operation_id !== operationId || slot.healthy_at))
|| !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) {
return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' };
}
const replacement = queueWatchdogAction(operation.server, 'rebuild', incidentId);
if (!replacement.ok) return replacement;
const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId;
if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason });
else recordDeploymentEvent(operationId, reason);
if (slot) db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(replacement.deploymentId, reason, operation.server, operationId);
recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server');
return replacement;
})();
}
src/lib/server/provisioning/restarts.ts
+23 -11
View File
@@ -1,13 +1,14 @@
import { config } from '../config.js';
import { db } from '../db/index.js';
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
import { rconStatusResult } from '../integrations/rcon.js';
import { probeServerStatus } from '../fleet/status-evidence.js';
import { deploymentStage, finishDeployment } from './history.js';
// Komodo queues restart work. Don't mistake the still-running old container
// for a completed restart when an API kick immediately follows the request.
const HEALTH_POLL_MS = 15_000;
const requestsInFlight = new Set<string>();
export const restartRequestInFlight = (server: string) => requestsInFlight.has(server);
interface RestartSlot {
server: string;
@@ -41,11 +42,14 @@ export function rolloutWaitReason(server: string, operationId = ''): string | nu
return 'Waiting for the current provisioning operation';
}
const slot = slotFor(server);
if (slot?.healthy_at === null) {
const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d
WHERE d.id = ? AND d.action IN ('restart','rebuild') AND EXISTS (
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId);
if (slot?.healthy_at === null && !ownHandoff) {
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
}
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
return n >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
}
export type RolloutRestartResult =
@@ -76,7 +80,7 @@ export async function restartInRollout(server: string, options: { operationId: s
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
if (result.ok) {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId);
deploymentStage(operationId!, 'Waiting for container and RCON health');
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)');
return { kind: 'started' };
}
// These failures cannot have queued a remote restart. Other failures
@@ -115,6 +119,13 @@ async function refresh(): Promise<void> {
.all() as RestartSlot[];
for (const slot of slots) {
if (requestsInFlight.has(slot.server)) continue;
// Recovery capacity transferred to a rebuild remains reserved until
// teardown. Its health is verified by the creation worker, never here.
if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND (
(d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health'))
OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`)
.get(slot.operation_id)) continue;
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
let error: string | null = null;
let players: number | null = null;
@@ -123,22 +134,23 @@ async function refresh(): Promise<void> {
if (!health.ok) {
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
} else {
const rcon = await rconStatusResult(slot.server);
if (!rcon.ok) error = `rcon not answering: ${rcon.reason}`;
else players = rcon.status.players;
const status = await probeServerStatus(slot.server, slot.requested_at);
if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`;
else players = status.players;
}
} catch (err) {
error = `health check: ${(err as Error).message}`;
}
const now = new Date().toISOString();
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at);
const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
if (!checked.changes) continue;
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) {
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
deploymentStage(slot.operation_id, 'Verified container and RCON health');
deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
finishDeployment(slot.operation_id, { ok: true });
}
}
src/lib/server/provisioning/worker.ts
+20 -9
View File
@@ -4,7 +4,7 @@ import { getServer } from '../fleet/servers.js';
import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
import { processCreations, processDestructions, rebuildServer } from './instances.js';
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { probePlayerCount } from '../fleet/status-evidence.js';
import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from './restarts.js';
import { cancelProvisioning } from './requests.js';
import type { RuntimeAction } from '../../provisioning.js';
@@ -12,14 +12,20 @@ import type { RuntimeAction } from '../../provisioning.js';
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
interface Source { kind: 'campaign' | 'beta' | 'maintenance'; source_id: string }
interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string }
const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[];
const priority = (sources: Source[]) => sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2;
const priority = (sources: Source[]) => sources.some(s => s.kind === 'watchdog') ? 2 : sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2;
/** Source work may hold or withdraw a request, but never executes it. */
function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' {
const sources = sourcesFor(operation.id);
if (!sources.length) return 'ready';
// Watchdog intent gates every linked source, including its drift campaign.
for (const source of sources.filter(source => source.kind === 'watchdog')) {
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(source.source_id, operation.server)
|| getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
if (db().prepare("SELECT 1 FROM settings WHERE key = 'watchdog_enabled' AND value = '0'").get()) return 'held';
}
let held = false;
for (const source of sources) {
if (source.kind === 'campaign') {
@@ -36,10 +42,10 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready';
if (row.outcome === 'paused') held = true;
} else {
} else if (source.kind === 'maintenance') {
const row = db().prepare('SELECT state, provisioning_id FROM maintenance_jobs WHERE id = ?').get(source.source_id) as { state: string; provisioning_id: string } | undefined;
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) return 'ready';
}
} else return 'ready';
}
return held ? 'held' : 'cancelled';
}
@@ -61,7 +67,8 @@ function blocker(server: string, id: string): string | null {
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ?
AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
LIMIT 1`).get(server)) return 'Waiting for campaign work on this server';
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
LIMIT 1`).get(server, id)) return 'Waiting for campaign work on this server';
return rolloutWaitReason(server, id);
}
@@ -93,14 +100,18 @@ async function run(): Promise<void> {
async function processOperation(operation: Operation): Promise<void> {
const { id, server, action } = operation;
// An explicit cleanup resume follows confirmed remote deletion. Complete
// that durable checkpoint even if the incident now needs attention or the
// Watchdog is paused; never strand the reserved replacement identity.
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
if (operation.state === 'waiting') {
const eligibility = sourceEligibility(operation);
if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
if (eligibility === 'held') { deploymentStage(id, 'Waiting for the requester to resume'); return; }
}
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
if (!komodoConfigured() || !rconConfigured()) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog');
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
if (operation.state === 'awaiting-health') {
const health = await komodoInspectHealthy(server);
if (isStopped(health.state)) {
@@ -114,7 +125,7 @@ async function processOperation(operation: Operation): Promise<void> {
if (reason) { deploymentStage(id, reason); return; }
const health = await komodoInspectHealthy(server);
if (action !== 'start' && !isStopped(health.state)) {
const players = await rconGetPlayerCount(server);
const players = await probePlayerCount(server);
db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
if (players !== 0) {
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);