josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Watchdog recovery and drift rebuild workflows

Browse Source
This commit is contained in:
josh
2026-09-27 15:13:38 -04:00
parent a7044f21a1
commit b9f5902945
36 changed files with 2131 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/worker.ts
+20 -9
View File
@@ -4,7 +4,7 @@ import { getServer } from '../fleet/servers.js';
import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
import { processCreations, processDestructions, rebuildServer } from './instances.js';
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { probePlayerCount } from '../fleet/status-evidence.js';
import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from './restarts.js';
import { cancelProvisioning } from './requests.js';
import type { RuntimeAction } from '../../provisioning.js';
@@ -12,14 +12,20 @@ import type { RuntimeAction } from '../../provisioning.js';
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
interface Source { kind: 'campaign' | 'beta' | 'maintenance'; source_id: string }
interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string }
const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[];
const priority = (sources: Source[]) => sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2;
const priority = (sources: Source[]) => sources.some(s => s.kind === 'watchdog') ? 2 : sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2;
/** Source work may hold or withdraw a request, but never executes it. */
function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' {
const sources = sourcesFor(operation.id);
if (!sources.length) return 'ready';
// Watchdog intent gates every linked source, including its drift campaign.
for (const source of sources.filter(source => source.kind === 'watchdog')) {
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(source.source_id, operation.server)
|| getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
if (db().prepare("SELECT 1 FROM settings WHERE key = 'watchdog_enabled' AND value = '0'").get()) return 'held';
}
let held = false;
for (const source of sources) {
if (source.kind === 'campaign') {
@@ -36,10 +42,10 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready';
if (row.outcome === 'paused') held = true;
} else {
} else if (source.kind === 'maintenance') {
const row = db().prepare('SELECT state, provisioning_id FROM maintenance_jobs WHERE id = ?').get(source.source_id) as { state: string; provisioning_id: string } | undefined;
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) return 'ready';
}
} else return 'ready';
}
return held ? 'held' : 'cancelled';
}
@@ -61,7 +67,8 @@ function blocker(server: string, id: string): string | null {
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ?
AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
LIMIT 1`).get(server)) return 'Waiting for campaign work on this server';
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
LIMIT 1`).get(server, id)) return 'Waiting for campaign work on this server';
return rolloutWaitReason(server, id);
}
@@ -93,14 +100,18 @@ async function run(): Promise<void> {
async function processOperation(operation: Operation): Promise<void> {
const { id, server, action } = operation;
// An explicit cleanup resume follows confirmed remote deletion. Complete
// that durable checkpoint even if the incident now needs attention or the
// Watchdog is paused; never strand the reserved replacement identity.
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
if (operation.state === 'waiting') {
const eligibility = sourceEligibility(operation);
if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
if (eligibility === 'held') { deploymentStage(id, 'Waiting for the requester to resume'); return; }
}
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
if (!komodoConfigured() || !rconConfigured()) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog');
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
if (operation.state === 'awaiting-health') {
const health = await komodoInspectHealthy(server);
if (isStopped(health.state)) {
@@ -114,7 +125,7 @@ async function processOperation(operation: Operation): Promise<void> {
if (reason) { deploymentStage(id, reason); return; }
const health = await komodoInspectHealthy(server);
if (action !== 'start' && !isStopped(health.state)) {
const players = await rconGetPlayerCount(server);
const players = await probePlayerCount(server);
db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
if (players !== 0) {
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);