josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Watchdog recovery and drift rebuild workflows

Browse Source
This commit is contained in:
josh
2026-09-27 15:13:38 -04:00
parent a7044f21a1
commit b9f5902945
36 changed files with 2131 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/instances.ts
+20 -13
View File
@@ -1,4 +1,4 @@
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
import { probePlayerCount, probeServerStatus } from '../fleet/status-evidence.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js';
@@ -229,7 +229,7 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
}
/** Reserve the same identity and port before removing any part of the instance. */
export function queueRebuildServer(name: string): CreateServerResult {
export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServerResult {
const server = getServer(name);
const game = gameOfServer(name);
const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
@@ -237,12 +237,12 @@ export function queueRebuildServer(name: string): CreateServerResult {
|| (server.host_dir && server.host_dir !== serverDir(name))) {
return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
}
if (!config.komodo.targetServer || !config.rcon.password) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
if (!config.komodo.targetServer || (!config.rcon.password && !triggeredBy.startsWith('watchdog:'))) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
// Validate that the replacement setup can be composed before destruction.
const recipe = captureRecipe(game, map.slug);
return db().transaction(() => {
const id = beginDeployment(name, 'rebuild');
const id = beginDeployment(name, 'rebuild', triggeredBy);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
@@ -436,6 +436,7 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
// Step 5: execute/Deploy — actually start the container.
deploymentStage(operation, 'Starting container');
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
const deployRes = await komodoDeploy(name);
if (!deployRes.ok) {
if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
@@ -541,7 +542,7 @@ async function runDestructions(): Promise<void> {
continue;
}
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
let players = await rconGetPlayerCount(operation.server);
let players = await probePlayerCount(operation.server);
if (players === null) {
const health = await komodoInspectHealthy(operation.server);
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
@@ -621,8 +622,13 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
// Test evidence cascades from runs; inventory and snapshots cascade from servers.
try {
const tx = db().transaction((n: string) => {
db().prepare('DELETE FROM deployment_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM deployment_files WHERE server = ?').run(n);
// The Watchdog's rebuild campaign observes this exact operation
// through the registry gap and retains its before/after file audit.
for (const table of ['deployment_servers', 'deployment_files']) {
db().prepare(`DELETE FROM ${table} WHERE server = ? AND deployment_id NOT IN (
SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL)`)
.run(n);
}
db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
@@ -647,6 +653,7 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
}
interface CreationJob {
deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
submitted_at?: string | null;
}
async function provisionCreation(id: string): Promise<CreateServerResult> {
@@ -739,7 +746,7 @@ async function runCreations(): Promise<CreationOutcomes> {
// Check only previously started servers. Newly launched servers keep their
// slot until a later pass, even if a health probe would answer immediately.
const checking = db().prepare(`SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
const checking = db().prepare(`SELECT j.*, COALESCE(d.submitted_at, d.started_at) AS submitted_at FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
for (const job of checking) {
const id = job.deployment_id;
@@ -751,14 +758,14 @@ async function runCreations(): Promise<CreationOutcomes> {
try {
const komodo = await komodoInspectHealthy(job.server);
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
const rcon = await rconStatusResult(job.server);
if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; }
const actualMap = rcon.status.map;
const status = await probeServerStatus(job.server, job.submitted_at);
if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason); continue; }
const actualMap = status.map;
if (actualMap !== job.expected_map) {
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map'));
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map'));
continue;
}
deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap);
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap);
finishDeployment(id, { ok: true });
} catch (err) {
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);