josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve deleted server history

Browse Source 
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.
This commit is contained in:
josh
2026-09-27 21:48:55 -04:00
parent 53b29f726c
commit 981fb27c1c
61 changed files with 929 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/maintenance/recovery.ts
+7 -6
View File
@@ -26,7 +26,7 @@ export function setWatchdogEnabled(enabled: boolean) {
}
function incident(id: string) { return db().prepare('SELECT * FROM watchdog_incidents WHERE id = ?').get(id) as WatchdogIncident | undefined; }
function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; }
function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND server_deleted_at IS NULL AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; }
function event(id: string, message: string) { db().prepare('INSERT INTO watchdog_events(incident_id,created_at,message) VALUES (?,?,?)').run(id, iso(), message); }
function transition(id: string, state: WatchdogState, summary: string) {
const previous = incident(id);
@@ -74,11 +74,11 @@ export function watchdogDetail(id: string): WatchdogDetail | null {
return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null;
}
function hasPending(server: string) {
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server);
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server);
}
function protectedWork(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)
|| !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.restart_state IN ('pending','held','failed','manual')
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)
|| !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','held','failed','manual')
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=ds.deployment_id) LIMIT 1`).get(server);
}
function addAction(row: WatchdogIncident, operationId: string, action: 'restart' | 'rebuild', campaignId: string | null = null) {
@@ -91,6 +91,7 @@ export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss')
return db().transaction(() => {
const row = incident(id);
if (!row) return { ok: false as const, status: 404, error: 'Watchdog incident not found.' };
if (row.server_deleted_at) return { ok: false as const, status: 409, error: 'This server was deleted; its recovery history cannot be changed.' };
if (actions(id).some(a => pendingStates.includes(a.operation_state ?? ''))) return { ok: false as const, status: 409, error: 'Provisioning is still active. Complete or cancel that operation first.' };
if (action === 'dismiss') {
if (!['open','attention'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only detected incidents or incidents needing attention can be dismissed.' };
@@ -106,7 +107,7 @@ export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss')
}
async function reconcileActions(): Promise<void> {
const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE state IN ('recovering','attention')").all() as WatchdogIncident[];
const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state IN ('recovering','attention')").all() as WatchdogIncident[];
for (const row of rows) {
const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1);
if (!action) continue;
@@ -229,7 +230,7 @@ async function run(forceScan: boolean): Promise<void> {
}
await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump));
// Preserve incidents for temporarily absent rebuild instances, but close retired registrations.
const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE state != 'resolved'").all() as WatchdogIncident[];
const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'").all() as WatchdogIncident[];
for (const row of abandoned) {
if (getServer(row.server) || hasPending(row.server)) continue;
const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined;
src/lib/server/maintenance/runtime.ts
+7 -6
View File
@@ -39,7 +39,7 @@ export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind']
if (getServer(server)?.desired_state === 'stopped') continue;
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes;
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting')").get(server) as { id: number };
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')").get(server) as { id: number };
requestRestart(server, { kind: 'maintenance', id: String(job.id) });
}
return added;
@@ -55,13 +55,14 @@ export function queueUpdates(servers?: string[], automatic = false): number {
if (servers && (!servers.length || servers.some(server => !versions.some(v => v.server === server)))) throw new Error('Choose existing fleet servers.');
const candidates = versions.filter(v => v.update_status === 'outdated' && (!servers || servers.includes(v.server)));
const names = candidates.filter(v => !automatic || !db().prepare(`SELECT 1 FROM maintenance_jobs
WHERE server = ? AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server);
WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server);
return names.length ? queueMaintenance(names, 'update', automatic ? 'automatic' : 'manual') : 0;
}
export function retryMaintenance(id: number): number {
const job = db().prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'").get(id) as MaintenanceJob | undefined;
if (!job) throw new Error('Only failed maintenance can be retried.');
if (job.server_deleted_at) throw new Error('This server was deleted; its maintenance history cannot be retried.');
if (!maintenanceAllowed()) throw new Error('Maintenance is paused.');
if (getServer(job.server)?.desired_state === 'stopped') throw new Error('This server is intentionally stopped.');
// Ambiguous lifecycle failures retain their slot and require diagnostics, not another request.
@@ -82,11 +83,11 @@ export function cancelMaintenance(id: number): boolean {
}
function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?')
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL')
.run(state, detail, nowIso(), job.id);
}
function hold(job: MaintenanceJob, detail: string) {
db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ?').run(detail, job.id);
db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL').run(detail, job.id);
}
let ticking: Promise<void> | null = null;
@@ -108,7 +109,7 @@ async function runTick(): Promise<void> {
writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) });
}
})();
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
for (const job of jobs) {
try { await processJob(job); }
catch { hold(job, 'Maintenance check failed; waiting for the next check.'); }
@@ -119,7 +120,7 @@ async function runTick(): Promise<void> {
}
async function processJob(job: MaintenanceJob): Promise<void> {
const server = getServer(job.server);
if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);