josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve deleted server history

Browse Source 
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.
This commit is contained in:
josh
2026-09-27 21:48:55 -04:00
parent 53b29f726c
commit 981fb27c1c
61 changed files with 929 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/worker.ts
+10 -10
View File
@@ -23,7 +23,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
if (!sources.length) return 'ready';
// Watchdog intent gates every linked source, including its drift campaign.
for (const source of sources.filter(source => source.kind === 'watchdog')) {
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(source.source_id, operation.server)
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(source.source_id, operation.server)
|| getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
const incident = db().prepare('SELECT kind FROM watchdog_incidents WHERE id = ?').get(source.source_id) as { kind: 'health' | 'drift' };
if (!maintenanceAllowed(incident.kind)) return 'held';
@@ -31,21 +31,21 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
let held = false;
for (const source of sources) {
if (source.kind === 'campaign') {
const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ?')
const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL')
.get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined;
if (row?.provisioning_id !== operation.id || !row.files_changed) continue;
if (['pending','failed'].includes(row.restart_state)) return 'ready';
if (['held','manual'].includes(row.restart_state)) held = true;
} else if (source.kind === 'beta') {
const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ?`)
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`)
.get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined;
if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue;
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready';
if (row.outcome === 'paused') held = true;
} else if (source.kind === 'maintenance') {
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ?').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) {
if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) return 'held';
@@ -59,7 +59,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
function blocker(server: string, id: string): string | null {
const sources = sourcesFor(id);
const rank = priority(sources);
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ?
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[];
const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number };
if (others.some(other => other.state !== 'waiting' || other.action === 'destroy'
@@ -69,8 +69,8 @@ function blocker(server: string, id: string): string | null {
// Beta activation must not wait on the campaign rows that the beta itself
// holds. Campaign/maintenance callers likewise never block on their own job.
if (rank > 0 && db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ?
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
@@ -93,7 +93,7 @@ async function run(): Promise<void> {
await processDestructions();
await refreshRestartRollout();
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
ORDER BY started_at, rowid`).all() as Operation[];
for (const operation of operations) {
try { await processOperation(operation); }
@@ -140,9 +140,9 @@ async function processOperation(operation: Operation): Promise<void> {
}
// Requests can arrive during the probes. Recheck before claiming the action.
if (blocker(server, id) || sourceEligibility(operation) !== 'ready') return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
if (action === 'rebuild') { await rebuildServer(id); return; }
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id);
if (!claim.changes) return;
if (action === 'start' || action === 'restart') {
const result = await restartInRollout(server, { operationId: id, action });