josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/maintenance/index.ts
+7 -2
View File
@@ -4,10 +4,15 @@ import { maintenanceTick, plutoniumStatus } from './runtime.js';
import type { MaintenanceOverview } from '../../maintenance.js';
export function maintenanceOverview(selectedJob: number | null = null): MaintenanceOverview {
return { policies: maintenancePolicies(), recovery: watchdogOverview(), runtime: plutoniumStatus(selectedJob) };
return {
policies: maintenancePolicies(),
recovery: watchdogOverview(),
runtime: plutoniumStatus(selectedJob)
};
}
export async function runMaintenance(): Promise<void> {
// One failing observer must not prevent the other capabilities from reconciling.
const results = await Promise.allSettled([maintenanceTick(), watchdogTick()]);
for (const result of results) if (result.status === 'rejected') console.error('[maintenance]', result.reason);
for (const result of results)
if (result.status === 'rejected') console.error('[maintenance]', result.reason);
}
src/lib/server/maintenance/policies.ts
+23 -6
View File
@@ -2,18 +2,31 @@ import { db } from '../db/index.js';
import type { MaintenancePolicy, MaintenancePolicies } from '../../maintenance.js';
function setting(key: string): string | undefined {
return (db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined)?.value;
return (
db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as
{ value: string } | undefined
)?.value;
}
function save(key: string, value: boolean) {
db().prepare('INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run(key, value ? '1' : '0');
db()
.prepare(
'INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value'
)
.run(key, value ? '1' : '0');
}
export function maintenancePolicies(): MaintenancePolicies {
// Preserve the old recovery switch without opting anyone into runtime upgrades.
const legacy = setting('watchdog_enabled') !== '0';
return {
paused: setting('maintenance_paused') === '1',
health: setting('maintenance_health') === undefined ? legacy : setting('maintenance_health') === '1',
drift: setting('maintenance_drift') === undefined ? legacy : setting('maintenance_drift') === '1',
health:
setting('maintenance_health') === undefined
? legacy
: setting('maintenance_health') === '1',
drift:
setting('maintenance_drift') === undefined
? legacy
: setting('maintenance_drift') === '1',
updates: setting('maintenance_updates') === '1'
};
}
@@ -21,5 +34,9 @@ export function maintenanceAllowed(policy?: MaintenancePolicy): boolean {
const policies = maintenancePolicies();
return !policies.paused && (!policy || policies[policy]);
}
export function setMaintenancePolicy(policy: MaintenancePolicy, enabled: boolean) { save(`maintenance_${policy}`, enabled); }
export function pauseMaintenance(paused: boolean) { save('maintenance_paused', paused); }
export function setMaintenancePolicy(policy: MaintenancePolicy, enabled: boolean) {
save(`maintenance_${policy}`, enabled);
}
export function pauseMaintenance(paused: boolean) {
save('maintenance_paused', paused);
}
src/lib/server/maintenance/recovery.ts
+556 -124
View File
@@ -3,15 +3,36 @@ import { db } from '../db/index.js';
import { nativeInstance } from '../backends/native.js';
import { getServer, listServers } from '../fleet/servers.js';
import { hasActiveProvisioning } from '../fleet/provisioning.js';
import { inspectForMonitoring, rconForMonitoring, monitoringStatus, monitoringIdentity, MonitoringChangedError } from '../fleet/health-observations.js';
import {
inspectForMonitoring,
rconForMonitoring,
monitoringStatus,
monitoringIdentity,
MonitoringChangedError
} from '../fleet/health-observations.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js';
import {
latestGameSubmission,
telemetryEvidence,
telemetryReport
} from '../fleet/status-evidence.js';
import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
import {
cancelProvisioning,
requestWatchdogLifecycle,
escalateWatchdogRestart
} from '../provisioning/requests.js';
import { deploymentStage, finishDeployment } from '../provisioning/history.js';
import { recordProgress } from '../provisioning/progress.js';
import { withOperationSource } from '../hosts/operation-context.js';
import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js';
import type {
WatchdogAction,
WatchdogDetail,
WatchdogEvidence,
WatchdogIncident,
WatchdogOverview,
WatchdogState
} from '../../watchdog.js';
import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
@@ -21,8 +42,18 @@ export const WATCHDOG_FAILURE_MS = 120_000;
export const WATCHDOG_RECOVERY_MS = 300_000;
const iso = () => new Date().toISOString();
const pendingStates = ['queued', 'waiting', 'running', 'awaiting-health'];
const setting = (key: string) => (db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined)?.value ?? null;
function setSetting(key: string, value: string) { db().prepare('INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run(key, value); }
const setting = (key: string) =>
(
db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as
{ value: string } | undefined
)?.value ?? null;
function setSetting(key: string, value: string) {
db()
.prepare(
'INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value'
)
.run(key, value);
}
export const watchdogEnabled = () => maintenanceAllowed('health') || maintenanceAllowed('drift');
export function setWatchdogEnabled(enabled: boolean) {
setSetting('watchdog_enabled', enabled ? '1' : '0');
@@ -30,18 +61,37 @@ export function setWatchdogEnabled(enabled: boolean) {
setMaintenancePolicy('drift', enabled);
}
function incident(id: string) { return db().prepare('SELECT * FROM watchdog_incidents WHERE id = ?').get(id) as WatchdogIncident | undefined; }
function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND server_deleted_at IS NULL AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; }
function event(id: string, message: string) { db().prepare('INSERT INTO watchdog_events(incident_id,created_at,message) VALUES (?,?,?)').run(id, iso(), message); }
function incident(id: string) {
return db().prepare('SELECT * FROM watchdog_incidents WHERE id = ?').get(id) as
WatchdogIncident | undefined;
}
function active(server: string, kind: 'health' | 'drift') {
return db()
.prepare(
"SELECT * FROM watchdog_incidents WHERE server = ? AND server_deleted_at IS NULL AND kind = ? AND state != 'resolved'"
)
.get(server, kind) as WatchdogIncident | undefined;
}
function event(id: string, message: string) {
db()
.prepare('INSERT INTO watchdog_events(incident_id,created_at,message) VALUES (?,?,?)')
.run(id, iso(), message);
}
function transition(id: string, state: WatchdogState, summary: string) {
const previous = incident(id);
if (!previous || (previous.state === state && previous.summary === summary)) return;
db().prepare('UPDATE watchdog_incidents SET state=?, summary=?, finished_at=? WHERE id=?')
db()
.prepare('UPDATE watchdog_incidents SET state=?, summary=?, finished_at=? WHERE id=?')
.run(state, summary, state === 'resolved' || state === 'dismissed' ? iso() : null, id);
event(id, summary);
}
function detect(server: string, kind: 'health' | 'drift', summary: string, details: WatchdogEvidence): WatchdogIncident {
const sampledAt = kind === 'health' ? details.observedAt ?? iso() : iso();
function detect(
server: string,
kind: 'health' | 'drift',
summary: string,
details: WatchdogEvidence
): WatchdogIncident {
const sampledAt = kind === 'health' ? (details.observedAt ?? iso()) : iso();
const sampledTime = Date.parse(sampledAt);
const previous = active(server, kind);
if (previous) {
@@ -49,40 +99,92 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai
const sampleGap = sampledTime - Date.parse(previous.last_seen_at);
const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable;
const prior = JSON.parse(previous.details) as WatchdogEvidence & { actionable?: boolean };
const newSample = sampleGap > 0
&& Date.now() - Date.parse(prior.countedAt ?? previous.last_seen_at) >= WATCHDOG_SCAN_MS;
const newSample =
sampleGap > 0 &&
Date.now() - Date.parse(prior.countedAt ?? previous.last_seen_at) >= WATCHDOG_SCAN_MS;
const priorActionable = prior.actionable;
// A successful routine/manual probe between watchdog samples breaks
// the failure streak, even if our next probe happens to fail again.
const after = latestGameSubmission(server);
const successful = [getServer(server)?.last_seen_at, telemetryReport(server)?.receivedAt]
.some(at => at && Date.parse(at) > Date.parse(previous.last_seen_at)
&& Date.parse(at) <= sampledTime && (!after || Date.parse(at) > Date.parse(after)));
const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true)
|| prior.identity !== details.identity || sampleGap > WATCHDOG_SCAN_MS + ONLINE_MAX_MS);
db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?')
.run(JSON.stringify({ ...details, countedAt: newSample || reset ? iso() : prior.countedAt ?? previous.last_seen_at }), newSample || reset ? sampledAt : previous.last_seen_at,
reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? sampledAt : previous.first_seen_at, previous.id);
const successful = [
getServer(server)?.last_seen_at,
telemetryReport(server)?.receivedAt
].some(
(at) =>
at &&
Date.parse(at) > Date.parse(previous.last_seen_at) &&
Date.parse(at) <= sampledTime &&
(!after || Date.parse(at) > Date.parse(after))
);
const reset =
kind === 'health' &&
(successful ||
(actionable === true && priorActionable !== true) ||
prior.identity !== details.identity ||
sampleGap > WATCHDOG_SCAN_MS + ONLINE_MAX_MS);
db()
.prepare(
'UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?'
)
.run(
JSON.stringify({
...details,
countedAt:
newSample || reset ? iso() : (prior.countedAt ?? previous.last_seen_at)
}),
newSample || reset ? sampledAt : previous.last_seen_at,
reset ? 1 : previous.failures + (newSample ? 1 : 0),
reset ? sampledAt : previous.first_seen_at,
previous.id
);
if (previous.state === 'open') transition(previous.id, 'open', summary);
return incident(previous.id)!;
}
const id = randomUUID();
db().prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
VALUES (?,?,?,'open',?,?,?,?)`).run(id, server, kind, summary, JSON.stringify({ ...details, countedAt: iso() }), sampledAt, sampledAt);
db()
.prepare(
`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
VALUES (?,?,?,'open',?,?,?,?)`
)
.run(
id,
server,
kind,
summary,
JSON.stringify({ ...details, countedAt: iso() }),
sampledAt,
sampledAt
);
event(id, summary);
return incident(id)!;
}
/** Call only after verifying container readiness and current game evidence. */
export function resolveRecoveredHealthOperation(operationId: string, players: number, source: 'rcon' | 'telemetry' | null): boolean {
export function resolveRecoveredHealthOperation(
operationId: string,
players: number,
source: 'rcon' | 'telemetry' | null
): boolean {
return db().transaction(() => {
const operation = db().prepare(`SELECT d.server, d.action, d.state, d.submitted_at, d.triggered_by, i.id AS incident_id
const operation = db()
.prepare(
`SELECT d.server, d.action, d.state, d.submitted_at, d.triggered_by, i.id AS incident_id
FROM server_deployments d JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
JOIN watchdog_incidents i ON i.id = s.source_id AND i.server = d.server
WHERE d.id = ? AND d.server_deleted_at IS NULL AND i.server_deleted_at IS NULL
AND i.kind = 'health' AND i.state IN ('open','recovering')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources other WHERE other.operation_id = d.id AND other.kind != 'watchdog')`)
.get(operationId) as { server: string; action: string; state: string; submitted_at: string | null; triggered_by: string; incident_id: string } | undefined;
AND NOT EXISTS (SELECT 1 FROM provisioning_sources other WHERE other.operation_id = d.id AND other.kind != 'watchdog')`
)
.get(operationId) as
| {
server: string;
action: string;
state: string;
submitted_at: string | null;
triggered_by: string;
incident_id: string;
}
| undefined;
if (!operation || getServer(operation.server)?.desired_state !== 'running') return false;
const waiting = operation.state === 'waiting' && operation.submitted_at === null;
if (waiting && !operation.triggered_by.startsWith('watchdog:')) return false;
@@ -90,17 +192,48 @@ export function resolveRecoveredHealthOperation(operationId: string, players: nu
// exists, and lets that work proceed. Submitted recovery must retain
// exclusive ownership before accepting evidence or releasing its slot.
if (!waiting && hasActiveProvisioning(operation.server, operationId)) return false;
if (!waiting && (operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state))) return false;
const message = waiting ? 'Server recovered before automatic recovery was submitted' : 'Fresh container and game health verified after restart';
if (
!waiting &&
(operation.action !== 'restart' ||
!['awaiting-health', 'failed'].includes(operation.state))
)
return false;
const message = waiting
? 'Server recovered before automatic recovery was submitted'
: 'Fresh container and game health verified after restart';
if (waiting && !cancelProvisioning(operationId, message)) return false;
const now = iso();
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = NULL
WHERE server = ? AND operation_id = ? AND healthy_at IS NULL`).run(now, now, players, operation.server, operationId);
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, operationId);
recordProgress(operationId, { blocker: null, observedAt: now,
db()
.prepare(
`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = NULL
WHERE server = ? AND operation_id = ? AND healthy_at IS NULL`
)
.run(now, now, players, operation.server, operationId);
db()
.prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?')
.run(now, players, operationId);
recordProgress(operationId, {
blocker: null,
observedAt: now,
recovery: { state: 'released', at: now, operationId, message },
checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', checkedAt: now },
{ key: 'game', label: 'Game status', source, result: 'pass', actual: players, checkedAt: now }] });
checks: [
{
key: 'container',
label: 'Container readiness',
source: 'agent',
result: 'pass',
checkedAt: now
},
{
key: 'game',
label: 'Game status',
source,
result: 'pass',
actual: players,
checkedAt: now
}
]
});
if (operation.state === 'awaiting-health') {
deploymentStage(operationId, message);
finishDeployment(operationId, { ok: true });
@@ -110,112 +243,278 @@ export function resolveRecoveredHealthOperation(operationId: string, players: nu
})();
}
function actions(id: string): WatchdogAction[] {
return db().prepare(`SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error
return db()
.prepare(
`SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error
FROM watchdog_actions a LEFT JOIN server_deployments d ON d.id=a.operation_id
WHERE a.incident_id=? ORDER BY a.rowid`).all(id) as WatchdogAction[];
WHERE a.incident_id=? ORDER BY a.rowid`
)
.all(id) as WatchdogAction[];
}
export function watchdogOverview(): WatchdogOverview {
return { enabled: watchdogEnabled(), lastScanAt: setting('watchdog_last_scan'), scanError: setting('watchdog_scan_error') || null,
incidents: db().prepare(`SELECT i.*, (SELECT d.stage FROM watchdog_actions a
return {
enabled: watchdogEnabled(),
lastScanAt: setting('watchdog_last_scan'),
scanError: setting('watchdog_scan_error') || null,
incidents: db()
.prepare(
`SELECT i.*, (SELECT d.stage FROM watchdog_actions a
JOIN server_deployments d ON d.id = a.operation_id WHERE a.incident_id = i.id AND a.cycle = i.cycle
ORDER BY a.rowid DESC LIMIT 1) AS operation_stage
FROM watchdog_incidents i WHERE state != 'resolved' OR id IN (
SELECT id FROM watchdog_incidents WHERE state = 'resolved' ORDER BY first_seen_at DESC, rowid DESC LIMIT 100)
ORDER BY CASE WHEN state='resolved' THEN 1 ELSE 0 END, first_seen_at DESC, rowid DESC`).all() as WatchdogIncident[] };
ORDER BY CASE WHEN state='resolved' THEN 1 ELSE 0 END, first_seen_at DESC, rowid DESC`
)
.all() as WatchdogIncident[]
};
}
export function watchdogDetail(id: string): WatchdogDetail | null {
const row = incident(id);
return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null;
return row
? {
incident: row,
actions: actions(id),
events: db()
.prepare(
'SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id'
)
.all(id) as WatchdogDetail['events']
}
: null;
}
function scanSuppressed(server: string, kind?: 'health' | 'drift'): boolean {
// An explicit retry can reclaim its own earlier attempt's recovery slot.
// Keep other findings and every active lifecycle operation suppressed.
const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r
const retained = db()
.prepare(
`SELECT r.operation_id FROM restart_rollout r
JOIN server_deployments d ON d.id = r.operation_id AND d.server_deleted_at IS NULL
JOIN watchdog_actions a ON a.operation_id = d.id
JOIN watchdog_incidents i ON i.id = a.incident_id
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
AND i.server = r.server AND i.server_deleted_at IS NULL AND i.state = 'open' AND a.cycle < i.cycle
AND (? IS NULL OR i.kind = ?) LIMIT 1`).get(server, kind ?? null, kind ?? null) as { operation_id: string } | undefined;
AND (? IS NULL OR i.kind = ?) LIMIT 1`
)
.get(server, kind ?? null, kind ?? null) as { operation_id: string } | undefined;
return hasActiveProvisioning(server, retained?.operation_id);
}
function protectedWork(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)
|| !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','held','failed','manual')
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=ds.deployment_id) LIMIT 1`).get(server);
return (
!!db()
.prepare(
`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`
)
.get(server) ||
!!db()
.prepare(
`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','held','failed','manual')
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=ds.deployment_id) LIMIT 1`
)
.get(server)
);
}
function addAction(row: WatchdogIncident, operationId: string, action: 'restart' | 'rebuild', campaignId: string | null = null) {
db().prepare('INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at,cycle) VALUES (?,?,?,?,?,?,?)')
function addAction(
row: WatchdogIncident,
operationId: string,
action: 'restart' | 'rebuild',
campaignId: string | null = null
) {
db()
.prepare(
'INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at,cycle) VALUES (?,?,?,?,?,?,?)'
)
.run(randomUUID(), row.id, action, operationId, campaignId, iso(), row.cycle);
transition(row.id, 'recovering', action === 'restart' ? 'Restart queued; waiting for safe recovery' : 'Rebuild queued with the latest promoted setup');
event(row.id, `${action === 'restart' ? 'Restart' : 'Rebuild'} requested: ${operationId}${campaignId ? `; campaign ${campaignId}` : ''}`);
transition(
row.id,
'recovering',
action === 'restart'
? 'Restart queued; waiting for safe recovery'
: 'Rebuild queued with the latest promoted setup'
);
event(
row.id,
`${action === 'restart' ? 'Restart' : 'Rebuild'} requested: ${operationId}${campaignId ? `; campaign ${campaignId}` : ''}`
);
}
export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss') {
return db().transaction(() => {
const row = incident(id);
if (!row) return { ok: false as const, status: 404, error: 'Watchdog incident not found.' };
if (row.server_deleted_at) return { ok: false as const, status: 409, error: 'This server was deleted; its recovery history cannot be changed.' };
if (actions(id).some(a => pendingStates.includes(a.operation_state ?? ''))) return { ok: false as const, status: 409, error: 'Provisioning is still active. Complete or cancel that operation first.' };
if (row.server_deleted_at)
return {
ok: false as const,
status: 409,
error: 'This server was deleted; its recovery history cannot be changed.'
};
if (actions(id).some((a) => pendingStates.includes(a.operation_state ?? '')))
return {
ok: false as const,
status: 409,
error: 'Provisioning is still active. Complete or cancel that operation first.'
};
if (action === 'dismiss') {
if (!['open','attention'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only detected incidents or incidents needing attention can be dismissed.' };
transition(id, 'dismissed', 'Dismissed by operator; automatic attempts remain suppressed until the condition clears');
if (!['open', 'attention'].includes(row.state))
return {
ok: false as const,
status: 409,
error: 'Only detected incidents or incidents needing attention can be dismissed.'
};
transition(
id,
'dismissed',
'Dismissed by operator; automatic attempts remain suppressed until the condition clears'
);
} else {
if (!['attention','dismissed'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only incidents needing attention or dismissed incidents can be retried.' };
if (!getServer(row.server)) return { ok: false as const, status: 409, error: 'The server is no longer registered; inspect its provisioning history.' };
db().prepare('UPDATE watchdog_incidents SET cycle=cycle+1, failures=1, first_seen_at=?, last_seen_at=? WHERE id=?').run(iso(), iso(), id);
transition(id, 'open', 'Operator requested a fresh recovery attempt; checking current conditions');
if (!['attention', 'dismissed'].includes(row.state))
return {
ok: false as const,
status: 409,
error: 'Only incidents needing attention or dismissed incidents can be retried.'
};
if (!getServer(row.server))
return {
ok: false as const,
status: 409,
error: 'The server is no longer registered; inspect its provisioning history.'
};
db()
.prepare(
'UPDATE watchdog_incidents SET cycle=cycle+1, failures=1, first_seen_at=?, last_seen_at=? WHERE id=?'
)
.run(iso(), iso(), id);
transition(
id,
'open',
'Operator requested a fresh recovery attempt; checking current conditions'
);
}
return { ok: true as const };
})();
}
async function reconcileActions(): Promise<void> {
const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state IN ('recovering','attention')").all() as WatchdogIncident[];
const rows = db()
.prepare(
"SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state IN ('recovering','attention')"
)
.all() as WatchdogIncident[];
for (const row of rows) {
const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1);
const action = actions(row.id)
.filter((a) => a.cycle === row.cycle)
.at(-1);
if (!action) continue;
// Continue observing our own recovery, but do not verify or escalate
// against files and health owned by another lifecycle operation.
if (hasActiveProvisioning(row.server, action.operation_id)) continue;
const operation = db().prepare('SELECT state,submitted_at,started_at,stage,error FROM server_deployments WHERE id=?').get(action.operation_id) as
{ state: string; submitted_at: string | null; started_at: string; stage: string; error: string | null } | undefined;
if (!operation) { transition(row.id, 'attention', 'Provisioning history is missing; inspect the server before retrying'); continue; }
if (['done','warning'].includes(operation.state)) {
const operation = db()
.prepare(
'SELECT state,submitted_at,started_at,stage,error FROM server_deployments WHERE id=?'
)
.get(action.operation_id) as
| {
state: string;
submitted_at: string | null;
started_at: string;
stage: string;
error: string | null;
}
| undefined;
if (!operation) {
transition(
row.id,
'attention',
'Provisioning history is missing; inspect the server before retrying'
);
continue;
}
if (['done', 'warning'].includes(operation.state)) {
if (row.kind === 'drift') {
try {
const plan = await previewDrift([row.server]);
if (incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue;
if (plan.files.length) { transition(row.id, 'attention', 'Rebuild completed but drift remains; inspect the campaign and server files'); continue; }
if (
incident(row.id)?.cycle !== row.cycle ||
hasActiveProvisioning(row.server, action.operation_id)
)
continue;
if (plan.files.length) {
transition(
row.id,
'attention',
'Rebuild completed but drift remains; inspect the campaign and server files'
);
continue;
}
} catch (err) {
if (!hasActiveProvisioning(row.server, action.operation_id))
transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`);
transition(
row.id,
'attention',
`Cannot verify rebuilt files: ${(err as Error).message}`
);
continue;
}
}
transition(row.id, 'resolved', action.action === 'restart' ? 'Server recovered after restart' : 'Server recovered after rebuild');
transition(
row.id,
'resolved',
action.action === 'restart'
? 'Server recovered after restart'
: 'Server recovered after rebuild'
);
continue;
}
if (row.state === 'attention') continue; // Exhausted attempts stay latched, including across app restarts.
if (['cancelled','interrupted'].includes(operation.state)) {
transition(row.id, 'attention', `Provisioning ${operation.state}; inspect the recorded operation before retrying`); continue;
}
if (action.action === 'rebuild') {
if (operation.state === 'failed') transition(row.id, 'attention', `Rebuild failed: ${operation.error ?? operation.stage}`);
else if (operation.state === 'awaiting-health' && Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) >= WATCHDOG_RECOVERY_MS)
transition(row.id, 'attention', 'Rebuild has not recovered within five minutes; provisioning continues verification');
if (['cancelled', 'interrupted'].includes(operation.state)) {
transition(
row.id,
'attention',
`Provisioning ${operation.state}; inspect the recorded operation before retrying`
);
continue;
}
if (!['awaiting-health','failed'].includes(operation.state)) continue;
if (Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) < WATCHDOG_RECOVERY_MS) continue;
if (action.action === 'rebuild') {
if (operation.state === 'failed')
transition(
row.id,
'attention',
`Rebuild failed: ${operation.error ?? operation.stage}`
);
else if (
operation.state === 'awaiting-health' &&
Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) >=
WATCHDOG_RECOVERY_MS
)
transition(
row.id,
'attention',
'Rebuild has not recovered within five minutes; provisioning continues verification'
);
continue;
}
if (!['awaiting-health', 'failed'].includes(operation.state)) continue;
if (
Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) <
WATCHDOG_RECOVERY_MS
)
continue;
if (!maintenanceAllowed(row.kind)) continue;
const current = getServer(row.server);
if (!current || current.desired_state === 'stopped' || protectedWork(row.server)) continue;
// A recovery deadline is a reason to check again, not evidence that
// rebuilding is necessary. The worker may not have polled this tick yet.
const observation = await observe(row.server, action.operation_id);
if (!observation || incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue;
if (
!observation ||
incident(row.id)?.cycle !== row.cycle ||
hasActiveProvisioning(row.server, action.operation_id)
)
continue;
if (observation.healthy) {
resolveRecoveredHealthOperation(action.operation_id, observation.players!, observation.source ?? null);
resolveRecoveredHealthOperation(
action.operation_id,
observation.players!,
observation.source ?? null
);
continue;
}
if (!observation.actionable) continue;
@@ -224,45 +523,110 @@ async function reconcileActions(): Promise<void> {
if (next.ok) addAction(row, next.deploymentId, 'rebuild');
return next;
})();
if (!result.ok && result.status !== 409 && result.status !== 503) transition(row.id, 'attention', `Could not queue rebuild: ${result.error}`);
if (!result.ok && result.status !== 409 && result.status !== 503)
transition(row.id, 'attention', `Could not queue rebuild: ${result.error}`);
}
}
interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; observedAt: string; identity: string; players?: number; source?: 'rcon' | 'telemetry' }
async function observe(server: string, operationId?: string, fresh = false): Promise<Observation | null> {
interface Observation {
healthy: boolean;
actionable: boolean;
reason: string;
containerState: string | null;
observedAt: string;
identity: string;
players?: number;
source?: 'rcon' | 'telemetry';
}
async function observe(
server: string,
operationId?: string,
fresh = false
): Promise<Observation | null> {
const epoch = latestGameSubmission(server);
const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server))
|| protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server);
const suppressed = () =>
(operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server)) ||
protectedWork(server) ||
getServer(server)?.desired_state !== 'running' ||
epoch !== latestGameSubmission(server);
// Routine watchdog passes only consume the status monitor. Manual checks and
// recovery verification still obtain fresh, independently verified evidence.
const snapshot = !fresh && !operationId ? monitoringStatus(server) : null;
if (!fresh && !operationId && !snapshot) return null;
const inspected = snapshot?.container ?? await inspectForMonitoring(server, true).catch(error => {
if (error instanceof MonitoringChangedError) return null;
throw error;
});
const inspected =
snapshot?.container ??
(await inspectForMonitoring(server, true).catch((error) => {
if (error instanceof MonitoringChangedError) return null;
throw error;
}));
if (!inspected) return null;
const health = inspected.result;
if (suppressed()) return null;
const identity = snapshot?.identity ?? monitoringIdentity(server);
const containerEvidence = { identity, observedAt: inspected.observedAt };
const state = health.state?.toLowerCase() ?? null;
if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['missing','stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
if (!health.ok)
return {
...containerEvidence,
healthy: false,
actionable:
!!state &&
[
'missing',
'stopped',
'exited',
'created',
'dead',
'unhealthy',
'restarting',
'starting'
].includes(state),
reason: health.reason,
containerState: state
};
const telemetry = telemetryEvidence(server);
if (telemetry) return { identity, observedAt: telemetry.receivedAt, healthy: true, actionable: false, reason: 'Container healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
const probed = snapshot ? snapshot.game : await rconForMonitoring(server, true).catch(error => {
if (error instanceof MonitoringChangedError) return null;
throw error;
});
if (telemetry)
return {
identity,
observedAt: telemetry.receivedAt,
healthy: true,
actionable: false,
reason: 'Container healthy with fresh telemetry',
containerState: state,
players: telemetry.players,
source: 'telemetry'
};
const probed = snapshot
? snapshot.game
: await rconForMonitoring(server, true).catch((error) => {
if (error instanceof MonitoringChangedError) return null;
throw error;
});
if (!probed) return null;
const rcon = probed.result;
if (suppressed()) return null;
const latest = telemetryEvidence(server);
const gameEvidence = { identity: snapshot?.identity ?? monitoringIdentity(server), observedAt: latest?.receivedAt ?? probed.observedAt };
if (rcon.ok || latest) return { ...gameEvidence, healthy: true, actionable: false, reason: 'Container healthy with fresh game status', containerState: state,
players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), source: latest ? 'telemetry' : 'rcon' };
return { ...gameEvidence, healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
const gameEvidence = {
identity: snapshot?.identity ?? monitoringIdentity(server),
observedAt: latest?.receivedAt ?? probed.observedAt
};
if (rcon.ok || latest)
return {
...gameEvidence,
healthy: true,
actionable: false,
reason: 'Container healthy with fresh game status',
containerState: state,
players: latest?.players ?? (rcon.ok ? rcon.status.players : 0),
source: latest ? 'telemetry' : 'rcon'
};
return {
...gameEvidence,
healthy: false,
actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`,
containerState: state
};
}
async function scanServer(server: string, forceScan: boolean): Promise<void> {
const row = getServer(server);
@@ -270,16 +634,31 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
if (scanSuppressed(server) || protectedWork(server)) return;
const observation = await observe(server, undefined, forceScan);
// Probes can overlap an API request; recheck ownership before queuing anything.
if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (
!observation ||
scanSuppressed(server) ||
protectedWork(server) ||
getServer(server)?.desired_state === 'stopped'
)
return;
if (observation.identity !== monitoringIdentity(server)) return;
if (!scanSuppressed(server, 'health') && observation.healthy) {
const current = active(server, 'health');
if (current) transition(current.id, 'resolved', 'Fresh game status and host agent health verified');
if (current)
transition(current.id, 'resolved', 'Fresh game status and host agent health verified');
} else if (!scanSuppressed(server, 'health')) {
const problem = detect(server, 'health', observation.reason, observation);
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
const oldEnough = Date.parse(observation.observedAt) - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
if (problem.state === 'open' && observation.actionable && oldEnough && problem.failures >= 3 && maintenanceAllowed('health')) {
const startup = ['starting', 'restarting'].includes(observation.containerState ?? '');
const oldEnough =
Date.parse(observation.observedAt) - Date.parse(problem.first_seen_at) >=
(startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
if (
problem.state === 'open' &&
observation.actionable &&
oldEnough &&
problem.failures >= 3 &&
maintenanceAllowed('health')
) {
// A missing container has nothing to restart. Recreate it through
// the normal rebuild queue, with its identity and safety checks.
const action = observation.containerState === 'missing' ? 'rebuild' : 'restart';
@@ -288,49 +667,85 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
if (next.ok) addAction(problem, next.deploymentId, action);
return next;
})();
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error);
if (!result.ok && result.status !== 409 && result.status !== 503)
transition(problem.id, 'attention', result.error);
}
}
// Missing-container recovery belongs to the health policy. Do not turn
// absent files into drift or bypass the sustained health-failure window.
// Unknown container health is an observation failure, not proof that a rebuild is safe.
if (observation.containerState === 'missing' || (!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
if (
observation.containerState === 'missing' ||
(!observation.healthy && !observation.actionable) ||
scanSuppressed(server, 'drift')
)
return;
// File trees change far less often than liveness. Persist the last successful
// scan per registration/process so health checks do not rescan every minute.
// Manual checks and new lifecycle submissions always get fresh file evidence.
const driftKey = `watchdog_drift_scan:${server}`;
const instance = nativeInstance(server);
const identity = JSON.stringify([row.created_at, instance?.id, instance?.generation, latestGameSubmission(server)]);
const identity = JSON.stringify([
row.created_at,
instance?.id,
instance?.generation,
latestGameSubmission(server)
]);
let previous: { identity?: string; at?: string } = {};
try { previous = JSON.parse(setting(driftKey) ?? '{}'); } catch { /* rescan legacy/invalid state */ }
try {
previous = JSON.parse(setting(driftKey) ?? '{}');
} catch {
/* rescan legacy/invalid state */
}
const age = Date.now() - Date.parse(previous?.at ?? '');
if (!forceScan && previous?.identity === identity && age >= 0 && age < WATCHDOG_DRIFT_SCAN_MS) return;
if (!forceScan && previous?.identity === identity && age >= 0 && age < WATCHDOG_DRIFT_SCAN_MS)
return;
const prepared = await prepareDriftCampaign(server);
if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (
scanSuppressed(server, 'drift') ||
protectedWork(server) ||
getServer(server)?.desired_state === 'stopped'
)
return;
setSetting(driftKey, JSON.stringify({ identity, at: iso() }));
if (!prepared.plan.files.length) {
const current = active(server, 'drift');
if (current) transition(current.id, 'resolved', 'Server files match the promoted setup');
return;
}
const problem = detect(server, 'drift', `${prepared.plan.files.length} file difference(s) from the promoted setup`, { files: prepared.plan.files, fingerprint: prepared.plan.fingerprint });
const problem = detect(
server,
'drift',
`${prepared.plan.files.length} file difference(s) from the promoted setup`,
{ files: prepared.plan.files, fingerprint: prepared.plan.fingerprint }
);
if (problem.state !== 'open' || !maintenanceAllowed('drift')) return;
const result = db().transaction(() => {
const next = requestWatchdogLifecycle(server, 'rebuild', problem.id);
if (next.ok) {
const campaignId = createDriftRebuildCampaign(server, next.deploymentId, problem.id, prepared);
const campaignId = createDriftRebuildCampaign(
server,
next.deploymentId,
problem.id,
prepared
);
addAction(problem, next.deploymentId, 'rebuild', campaignId);
}
return next;
})();
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', `Could not queue drift rebuild: ${result.error}`);
if (!result.ok && result.status !== 409 && result.status !== 503)
transition(problem.id, 'attention', `Could not queue drift rebuild: ${result.error}`);
}
let inFlight: Promise<void> | null = null;
export function watchdogTick(forceScan = false): Promise<void> {
if (inFlight) return inFlight;
inFlight = withOperationSource(forceScan ? 'Manual maintenance check' : 'Maintenance watchdog', () => run(forceScan))
.finally(() => { inFlight = null; });
inFlight = withOperationSource(
forceScan ? 'Manual maintenance check' : 'Maintenance watchdog',
() => run(forceScan)
).finally(() => {
inFlight = null;
});
return inFlight;
}
async function run(forceScan: boolean): Promise<void> {
@@ -338,8 +753,12 @@ async function run(forceScan: boolean): Promise<void> {
const lastScan = setting('watchdog_last_scan');
if (!forceScan && lastScan && Date.now() - Date.parse(lastScan) < WATCHDOG_SCAN_MS) return;
setSetting('watchdog_last_scan', iso());
if (!listServers().some(server => nativeInstance(server.name))) {
setSetting('watchdog_scan_error', 'Register a native host instance before automatic recovery. RCON or fresh telemetry verifies game status.'); return;
if (!listServers().some((server) => nativeInstance(server.name))) {
setSetting(
'watchdog_scan_error',
'Register a native host instance before automatic recovery. RCON or fresh telemetry verifies game status.'
);
return;
}
const errors: string[] = [];
const servers = listServers();
@@ -347,21 +766,34 @@ async function run(forceScan: boolean): Promise<void> {
async function pump() {
while (cursor < servers.length) {
const server = servers[cursor++];
try { await scanServer(server.name, forceScan); }
catch (err) {
if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`);
try {
await scanServer(server.name, forceScan);
} catch (err) {
if (!hasActiveProvisioning(server.name))
errors.push(`${server.name}: ${(err as Error).message}`);
}
}
}
await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump));
// Preserve incidents for temporarily absent rebuild instances, but close retired registrations.
const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'").all() as WatchdogIncident[];
const abandoned = db()
.prepare(
"SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'"
)
.all() as WatchdogIncident[];
for (const row of abandoned) {
if (getServer(row.server) || hasActiveProvisioning(row.server)) continue;
const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined;
const last = db()
.prepare(
'SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1'
)
.get(row.server) as { action: string; state: string } | undefined;
// An absent registry can also mean a failed rebuild. Only a completed
// destruction retires that failure without requiring inspection.
if (['open','dismissed'].includes(row.state) || (last?.action === 'destroy' && ['done','warning'].includes(last.state)))
if (
['open', 'dismissed'].includes(row.state) ||
(last?.action === 'destroy' && ['done', 'warning'].includes(last.state))
)
transition(row.id, 'resolved', 'Server is no longer registered');
}
setSetting('watchdog_scan_error', errors.join('\n'));
src/lib/server/maintenance/runtime.ts
+171 -47
View File
@@ -6,42 +6,77 @@ import { nextMaintenanceAt } from '../../plutonium.js';
import type { MaintenanceJob, MaintenanceSchedule, PlutoniumStatus } from '../../plutonium.js';
import { readSetting, writeSetting } from './settings.js';
import { checkPlutonium, probeVersion, release, freshRelease, runningVersions, HOUR } from '../integrations/plutonium.js';
import {
checkPlutonium,
probeVersion,
release,
freshRelease,
runningVersions,
HOUR
} from '../integrations/plutonium.js';
import { maintenanceAllowed } from './policies.js';
export { checkPlutonium } from '../integrations/plutonium.js';
const nowIso = () => new Date().toISOString();
function schedule(): MaintenanceSchedule {
// Legacy schedules were explicitly UTC. Preserve their timing until saved.
return { timeZone: 'UTC', ...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null }) };
return {
timeZone: 'UTC',
...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null })
};
}
export function saveMaintenanceSchedule(enabled: boolean, time: string, timeZone: string): void {
const nextAt = nextMaintenanceAt(time, new Date(), timeZone);
writeSetting('plutonium_schedule', { enabled, time, timeZone, nextAt: enabled ? nextAt : null });
writeSetting('plutonium_schedule', {
enabled,
time,
timeZone,
nextAt: enabled ? nextAt : null
});
}
export function plutoniumStatus(selectedJob: number | null = null): PlutoniumStatus {
return {
release: release(), schedule: schedule(), servers: runningVersions(),
jobs: db().prepare(`SELECT m.*, d.stage AS operation_stage, d.state AS operation_state
release: release(),
schedule: schedule(),
servers: runningVersions(),
jobs: db()
.prepare(
`SELECT m.*, d.stage AS operation_stage, d.state AS operation_state
FROM maintenance_jobs m LEFT JOIN server_deployments d ON d.id = m.provisioning_id
WHERE m.state IN ('queued','restarting')
OR (m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id))
OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) OR m.id = ? ORDER BY m.id DESC`).all(selectedJob) as MaintenanceJob[]
OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) OR m.id = ? ORDER BY m.id DESC`
)
.all(selectedJob) as MaintenanceJob[]
};
}
export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind'] = 'legacy', origin: MaintenanceJob['origin'] = 'manual'): number {
const names = new Set(listServers().map(s => s.name));
if (!servers.length || servers.some(s => !names.has(s))) throw new Error('Choose existing fleet servers.');
export function queueMaintenance(
servers: string[],
kind: MaintenanceJob['kind'] = 'legacy',
origin: MaintenanceJob['origin'] = 'manual'
): number {
const names = new Set(listServers().map((s) => s.name));
if (!servers.length || servers.some((s) => !names.has(s)))
throw new Error('Choose existing fleet servers.');
return db().transaction(() => {
const target = kind === 'schedule' ? null : freshRelease();
if (kind === 'update' && target === null) throw new Error('Check the published revision before requesting an update.');
if (kind === 'update' && target === null)
throw new Error('Check the published revision before requesting an update.');
let added = 0;
for (const server of new Set(servers)) {
if (getServer(server)?.desired_state === 'stopped') continue;
if (origin !== 'manual' && hasActiveProvisioning(server)) continue;
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes;
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')").get(server) as { id: number };
added += db()
.prepare(
`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
VALUES (?, ?, ?, ?, ?)`
)
.run(server, target, nowIso(), kind, origin).changes;
const job = db()
.prepare(
"SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')"
)
.get(server) as { id: number };
requestRestart(server, { kind: 'maintenance', id: String(job.id) });
}
return added;
@@ -50,52 +85,98 @@ export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind']
/** One automatic attempt per server/revision. Explicit retries preserve the old outcome. */
export function queueUpdates(servers?: string[], automatic = false): number {
if (!maintenanceAllowed()) throw new Error('Maintenance is paused. Resume it before requesting updates.');
if (!maintenanceAllowed())
throw new Error('Maintenance is paused. Resume it before requesting updates.');
if (automatic && !maintenanceAllowed('updates')) return 0;
if (freshRelease() === null) throw new Error('Check the published revision before requesting an update.');
if (freshRelease() === null)
throw new Error('Check the published revision before requesting an update.');
const versions = runningVersions();
if (servers && (!servers.length || servers.some(server => !versions.some(v => v.server === server)))) throw new Error('Choose existing fleet servers.');
const candidates = versions.filter(v => v.update_status === 'outdated' && (!servers || servers.includes(v.server)));
const names = candidates.filter(v => !automatic || !db().prepare(`SELECT 1 FROM maintenance_jobs
WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server);
if (
servers &&
(!servers.length || servers.some((server) => !versions.some((v) => v.server === server)))
)
throw new Error('Choose existing fleet servers.');
const candidates = versions.filter(
(v) => v.update_status === 'outdated' && (!servers || servers.includes(v.server))
);
const names = candidates
.filter(
(v) =>
!automatic ||
!db()
.prepare(
`SELECT 1 FROM maintenance_jobs
WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'`
)
.get(v.server, freshRelease())
)
.map((v) => v.server);
return names.length ? queueMaintenance(names, 'update', automatic ? 'automatic' : 'manual') : 0;
}
export function retryMaintenance(id: number): number {
const job = db().prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'").get(id) as MaintenanceJob | undefined;
const job = db()
.prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'")
.get(id) as MaintenanceJob | undefined;
if (!job) throw new Error('Only failed maintenance can be retried.');
if (job.server_deleted_at) throw new Error('This server was deleted; its maintenance history cannot be retried.');
if (job.server_deleted_at)
throw new Error('This server was deleted; its maintenance history cannot be retried.');
if (!maintenanceAllowed()) throw new Error('Maintenance is paused.');
if (getServer(job.server)?.desired_state === 'stopped') throw new Error('This server is intentionally stopped.');
if (getServer(job.server)?.desired_state === 'stopped')
throw new Error('This server is intentionally stopped.');
// Ambiguous lifecycle failures retain their slot and require diagnostics, not another request.
if (job.provisioning_id) {
const operation = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(job.provisioning_id) as { state: string } | undefined;
if (operation?.state !== 'done') throw new Error('Inspect and recover the existing Provisioning operation before retrying.');
const operation = db()
.prepare('SELECT state FROM server_deployments WHERE id = ?')
.get(job.provisioning_id) as { state: string } | undefined;
if (operation?.state !== 'done')
throw new Error(
'Inspect and recover the existing Provisioning operation before retrying.'
);
}
return queueMaintenance([job.server], job.kind, 'manual');
}
export function cancelMaintenance(id: number): boolean {
return db().transaction(() => {
const job = db().prepare("SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'").get(id) as { provisioning_id: string | null } | undefined;
const job = db()
.prepare(
"SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'"
)
.get(id) as { provisioning_id: string | null } | undefined;
if (!job) return false;
if (job.provisioning_id && !cancelProvisioning(job.provisioning_id)) return false;
return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0;
return (
db()
.prepare(
`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
WHERE id = ? AND state = 'queued'`
)
.run(nowIso(), id).changes > 0
);
})();
}
function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL')
db()
.prepare(
'UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL'
)
.run(state, detail, nowIso(), job.id);
}
function hold(job: MaintenanceJob, detail: string) {
db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL').run(detail, job.id);
db()
.prepare(
'UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL'
)
.run(detail, job.id);
}
let ticking: Promise<void> | null = null;
export function maintenanceTick(): Promise<void> {
if (ticking) return ticking;
ticking = runTick().finally(() => { ticking = null; });
ticking = runTick().finally(() => {
ticking = null;
});
return ticking;
}
async function runTick(): Promise<void> {
@@ -103,51 +184,94 @@ async function runTick(): Promise<void> {
// Provisioning refreshes the manifest independently; that must not postpone
// the hourly version checks for the rest of the fleet.
const versionsCheckedAt = readSetting<string | null>('plutonium_versions_checked_at', null);
if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR
|| !versionsCheckedAt || Date.now() - Date.parse(versionsCheckedAt) >= HOUR) await checkPlutonium();
if (
!latest.checkedAt ||
Date.now() - Date.parse(latest.checkedAt) >= HOUR ||
!versionsCheckedAt ||
Date.now() - Date.parse(versionsCheckedAt) >= HOUR
)
await checkPlutonium();
if (maintenanceAllowed('updates') && freshRelease() !== null) queueUpdates(undefined, true);
db().transaction(() => {
const plan = schedule();
if (maintenanceAllowed() && plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) {
const names = listServers().filter(s => s.desired_state !== 'stopped').map(s => s.name);
const names = listServers()
.filter((s) => s.desired_state !== 'stopped')
.map((s) => s.name);
if (names.length) queueMaintenance(names, 'schedule', 'schedule');
// Coalesce missed days after downtime into one run; no restart backlog.
writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) });
writeSetting('plutonium_schedule', {
...plan,
nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone)
});
}
})();
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
const jobs = db()
.prepare(
`SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id`
)
.all() as MaintenanceJob[];
for (const job of jobs) {
try { await processJob(job); }
catch { hold(job, 'Maintenance check failed; waiting for the next check.'); }
try {
await processJob(job);
} catch {
hold(job, 'Maintenance check failed; waiting for the next check.');
}
}
}
async function processJob(job: MaintenanceJob): Promise<void> {
const server = getServer(job.server);
if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
if (job.server_deleted_at || !server || server.destroyed_at)
return finish(job, 'cancelled', 'Server removed.');
if (!job.provisioning_id && !job.restarted_at && hasActiveProvisioning(job.server)) {
return hold(job, 'Waiting for another provisioning operation.');
}
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);
if (['failed', 'interrupted', 'cancelled'].includes(operation.state)) {
return finish(
job,
operation.state === 'cancelled' ? 'cancelled' : 'failed',
operation.error ?? operation.stage
);
}
if (operation.state !== 'done') {
if (operation.submitted_at) {
db().prepare("UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?").run(operation.submitted_at, job.id);
db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?').run('Waiting for restart recovery.', job.server);
db()
.prepare(
"UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?"
)
.run(operation.submitted_at, job.id);
db()
.prepare(
'UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?'
)
.run('Waiting for restart recovery.', job.server);
}
return hold(job, operation.stage);
}
if (hasActiveProvisioning(job.server, operation.id)) {
return hold(job, 'Waiting for another provisioning operation before verifying the running revision.');
return hold(
job,
'Waiting for another provisioning operation before verifying the running revision.'
);
}
const revision = await probeVersion(job.server, { maintenance: true });
if (hasActiveProvisioning(job.server, operation.id)) {
return hold(job, 'Waiting for another provisioning operation before verifying the running revision.');
return hold(
job,
'Waiting for another provisioning operation before verifying the running revision.'
);
}
if (job.target_revision && (revision === null || revision < job.target_revision)) {
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
return finish(
job,
'failed',
`Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`
);
}
return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`);
return finish(
job,
'done',
`Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`
);
}
src/lib/server/maintenance/settings.ts
+6 -2
View File
@@ -1,9 +1,13 @@
import { db } from '../db/index.js';
export function readSetting<T>(key: string, fallback: T): T {
const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined;
const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as
{ value: string } | undefined;
return row ? JSON.parse(row.value) : fallback;
}
export function writeSetting(key: string, value: unknown): void {
db().prepare('INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value')
db()
.prepare(
'INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value'
)
.run(key, JSON.stringify(value));
}