josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Suppress maintenance watchers during provisioning

Browse Source 
Keep maintenance observations and automatic requests clear of active lifecycle operations and retained recovery capacity while preserving owned recovery and explicit retries.

Fixes #55
This commit is contained in:
josh
2026-10-01 11:54:22 -04:00
parent a3cd3ee585
commit 0cb48c435e
8 changed files with 366 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/provisioning.ts
+9
View File
@@ -1,5 +1,14 @@
import { db } from '../db/index.js';
/** Maintenance must leave pending lifecycle work and retained recovery to Provisioning. */
export function hasActiveProvisioning(server: string, exceptOperationId = ''): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL AND id != ?
AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1`).get(server, exceptOperationId)
|| !!db().prepare(`SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL
AND (operation_id IS NULL OR operation_id != ?)`).get(server, exceptOperationId);
}
/** Only the latest lifecycle operation can describe the current server. */
export function isProvisioning(server: string): boolean {
const operation = db().prepare(`SELECT action, state FROM server_deployments
src/lib/server/integrations/plutonium.ts
+9 -2
View File
@@ -1,6 +1,7 @@
import { db } from '../db/index.js';
import { config } from '../config.js';
import { getServer, listServers } from '../fleet/servers.js';
import { hasActiveProvisioning } from '../fleet/provisioning.js';
import { latestGameSubmission } from '../fleet/status-evidence.js';
import { rconPlutoniumVersion } from './rcon.js';
import { komodoInspectHealthy } from './komodo.js';
@@ -34,15 +35,21 @@ export function runningVersions(): PlutoniumVersion[] {
: revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' };
});
}
export async function probeVersion(server: string): Promise<number | null> {
export async function probeVersion(server: string, options: { maintenance?: boolean } = {}): Promise<number | null> {
// Startup verification calls this directly; only maintenance observers yield
// ownership, without replacing provisioning's version evidence with an error.
const suppressed = () => options.maintenance && hasActiveProvisioning(server);
if (suppressed()) return null;
const original = getServer(server);
if (!original) return null;
const epoch = latestGameSubmission(server);
const recovering = restartRecoveryPending(server);
const health = !recovering && original.desired_state !== 'stopped'
? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false };
if (suppressed()) return null;
let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' }
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
if (suppressed()) return null;
const current = getServer(server);
// An older probe must not clear evidence already recorded for a new process
// or replacement registration, including replacements with no submission yet.
@@ -72,7 +79,7 @@ async function check(): Promise<void> {
await checkRelease();
const servers = listServers();
for (let i = 0; i < servers.length; i += 4) {
await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name)));
await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name, { maintenance: true })));
}
writeSetting('plutonium_versions_checked_at', nowIso());
}
src/lib/server/maintenance/recovery.ts
+34 -14
View File
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { komodoConfigured } from '../config.js';
import { getServer, listServers } from '../fleet/servers.js';
import { hasActiveProvisioning } from '../fleet/provisioning.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { rconStatusResult } from '../integrations/rcon.js';
import { telemetryEvidence } from '../fleet/status-evidence.js';
@@ -73,8 +74,17 @@ export function watchdogDetail(id: string): WatchdogDetail | null {
const row = incident(id);
return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null;
}
function hasPending(server: string) {
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server);
function scanSuppressed(server: string, kind?: 'health' | 'drift'): boolean {
// An explicit retry can reclaim its own earlier attempt's recovery slot.
// Keep other findings and every active lifecycle operation suppressed.
const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r
JOIN server_deployments d ON d.id = r.operation_id AND d.server_deleted_at IS NULL
JOIN watchdog_actions a ON a.operation_id = d.id
JOIN watchdog_incidents i ON i.id = a.incident_id
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
AND i.server = r.server AND i.server_deleted_at IS NULL AND i.state = 'open' AND a.cycle < i.cycle
AND (? IS NULL OR i.kind = ?) LIMIT 1`).get(server, kind ?? null, kind ?? null) as { operation_id: string } | undefined;
return hasActiveProvisioning(server, retained?.operation_id);
}
function protectedWork(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)
@@ -111,6 +121,9 @@ async function reconcileActions(): Promise<void> {
for (const row of rows) {
const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1);
if (!action) continue;
// Continue observing our own recovery, but do not verify or escalate
// against files and health owned by another lifecycle operation.
if (hasActiveProvisioning(row.server, action.operation_id)) continue;
const operation = db().prepare('SELECT state,submitted_at,started_at,stage,error FROM server_deployments WHERE id=?').get(action.operation_id) as
{ state: string; submitted_at: string | null; started_at: string; stage: string; error: string | null } | undefined;
if (!operation) { transition(row.id, 'attention', 'Provisioning history is missing; inspect the server before retrying'); continue; }
@@ -118,9 +131,13 @@ async function reconcileActions(): Promise<void> {
if (row.kind === 'drift') {
try {
const plan = await previewDrift([row.server]);
if (incident(row.id)?.cycle !== row.cycle) continue;
if (incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue;
if (plan.files.length) { transition(row.id, 'attention', 'Rebuild completed but drift remains; inspect the campaign and server files'); continue; }
} catch (err) { transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`); continue; }
} catch (err) {
if (!hasActiveProvisioning(row.server, action.operation_id))
transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`);
continue;
}
}
transition(row.id, 'resolved', action.action === 'restart' ? 'Server recovered after restart' : 'Server recovered after rebuild');
continue;
@@ -150,8 +167,9 @@ async function reconcileActions(): Promise<void> {
}
interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null }
async function observe(server: string): Promise<Observation> {
async function observe(server: string): Promise<Observation | null> {
const health = await komodoInspectHealthy(server);
if (scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return null;
const state = health.state?.toLowerCase() ?? null;
if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
if (telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state };
@@ -162,14 +180,14 @@ async function observe(server: string): Promise<Observation> {
async function scanServer(server: string): Promise<void> {
const row = getServer(server);
if (!row || row.desired_state === 'stopped') return;
if (hasPending(server) || protectedWork(server)) return;
if (scanSuppressed(server) || protectedWork(server)) return;
const observation = await observe(server);
// Probes can overlap an API request; recheck ownership before queuing anything.
if (hasPending(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (observation.healthy) {
if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (!scanSuppressed(server, 'health') && observation.healthy) {
const current = active(server, 'health');
if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified');
} else {
} else if (!scanSuppressed(server, 'health')) {
const problem = detect(server, 'health', observation.reason, observation);
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
const oldEnough = Date.now() - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
@@ -181,11 +199,11 @@ async function scanServer(server: string): Promise<void> {
})();
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error);
}
// Unknown container health is an observation failure, not proof that a rebuild is safe.
if (!observation.actionable || hasPending(server)) return;
}
// Unknown container health is an observation failure, not proof that a rebuild is safe.
if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
const prepared = await prepareDriftCampaign(server);
if (hasPending(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (!prepared.plan.files.length) {
const current = active(server, 'drift');
if (current) transition(current.id, 'resolved', 'Server files match the promoted setup');
@@ -225,14 +243,16 @@ async function run(forceScan: boolean): Promise<void> {
while (cursor < servers.length) {
const server = servers[cursor++];
try { await scanServer(server.name); }
catch (err) { errors.push(`${server.name}: ${(err as Error).message}`); }
catch (err) {
if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`);
}
}
}
await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump));
// Preserve incidents for temporarily absent rebuild instances, but close retired registrations.
const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'").all() as WatchdogIncident[];
for (const row of abandoned) {
if (getServer(row.server) || hasPending(row.server)) continue;
if (getServer(row.server) || hasActiveProvisioning(row.server)) continue;
const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined;
// An absent registry can also mean a failed rebuild. Only a completed
// destruction retires that failure without requiring inspection.
src/lib/server/maintenance/runtime.ts
+12 -1
View File
@@ -1,5 +1,6 @@
import { db } from '../db/index.js';
import { getServer, listServers } from '../fleet/servers.js';
import { hasActiveProvisioning } from '../fleet/provisioning.js';
import { requestRestart, cancelProvisioning } from '../provisioning/index.js';
import { nextMaintenanceAt } from '../../plutonium.js';
import type { MaintenanceJob, MaintenanceSchedule, PlutoniumStatus } from '../../plutonium.js';
@@ -37,6 +38,7 @@ export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind']
let added = 0;
for (const server of new Set(servers)) {
if (getServer(server)?.desired_state === 'stopped') continue;
if (origin !== 'manual' && hasActiveProvisioning(server)) continue;
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes;
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')").get(server) as { id: number };
@@ -125,6 +127,9 @@ async function runTick(): Promise<void> {
async function processJob(job: MaintenanceJob): Promise<void> {
const server = getServer(job.server);
if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
if (!job.provisioning_id && !job.restarted_at && hasActiveProvisioning(job.server)) {
return hold(job, 'Waiting for another provisioning operation.');
}
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);
@@ -136,7 +141,13 @@ async function processJob(job: MaintenanceJob): Promise<void> {
}
return hold(job, operation.stage);
}
const revision = await probeVersion(job.server);
if (hasActiveProvisioning(job.server, operation.id)) {
return hold(job, 'Waiting for another provisioning operation before verifying the running revision.');
}
const revision = await probeVersion(job.server, { maintenance: true });
if (hasActiveProvisioning(job.server, operation.id)) {
return hold(job, 'Waiting for another provisioning operation before verifying the running revision.');
}
if (job.target_revision && (revision === null || revision < job.target_revision)) {
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
}