diff --git a/README.md b/README.md index ce43293..67fe7a5 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,9 @@ Maintenance decides when work is needed and verifies the result. Provisioning executes every lifecycle operation using the shared safety checks and queues. Health recovery uses one restart followed by one rebuild if needed; drift repair requests a rebuild with a linked Campaign. **Rebuild deletes all instance files.** +Maintenance skips observations and new automatic work on servers with active +Provisioning operations or retained recovery capacity. Other servers continue +normally, and existing maintenance still tracks its own operation's outcome. Plutonium updates are applied by Plutainer's startup updater (`PLUTAINER_AUTO_UPDATE` must be enabled). Check versions and choose **Update on @@ -109,6 +112,7 @@ Optional daily restarts are independent of updates. Schedules use the saved timezone, follow daylight saving changes, and coalesce missed days into one run. Intentionally stopped servers are excluded. Existing schedules, pending jobs, operation IDs, and history survive the move from Settings and Watchdog. +Servers owned by Provisioning are skipped for that scheduled occurrence. **Pause all maintenance** holds new and queued maintenance while checks and submitted recovery continue. Each capability retains its own policy. Automatic diff --git a/docs/maintenance.md b/docs/maintenance.md index f73fdb4..e086607 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -41,6 +41,13 @@ future runs but does not cancel requests it already created. All capabilities respect intentionally stopped servers. Start them explicitly through Provisioning before expecting maintenance to act on them. +While Provisioning owns a server, Maintenance skips new health/drift observations, +version probes, automatic updates, and scheduled restart requests for that server. +Queued, waiting, running, and recovering operations count, including failed +operations that retain recovery capacity. Other servers continue normally. Existing +maintenance tracks its own operation to completion; revision verification waits +until any later operation releases the server. + ## Plutonium updates The published revision comes from the official production updater manifest. @@ -78,7 +85,9 @@ that same revision; an operator can request an update explicitly. Daily restarts are independent of update availability. At the saved queue time, Maintenance requests a restart for each server whose desired state is running. -These requests have no revision requirement. Busy servers can restart later. +These requests have no revision requirement. Servers owned by Provisioning are +skipped for that occurrence; the schedule advances normally. Requests already +queued can wait for players, campaign work, or beta work to finish. Saving a schedule uses the selected timezone and follows daylight saving changes. Skipped times run at the first valid minute after the gap; repeated times run once diff --git a/src/lib/server/fleet/provisioning.ts b/src/lib/server/fleet/provisioning.ts index 3f8eff2..d76bb49 100644 --- a/src/lib/server/fleet/provisioning.ts +++ b/src/lib/server/fleet/provisioning.ts @@ -1,5 +1,14 @@ import { db } from '../db/index.js'; +/** Maintenance must leave pending lifecycle work and retained recovery to Provisioning. */ +export function hasActiveProvisioning(server: string, exceptOperationId = ''): boolean { + return !!db().prepare(`SELECT 1 FROM server_deployments + WHERE server = ? AND server_deleted_at IS NULL AND id != ? + AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1`).get(server, exceptOperationId) + || !!db().prepare(`SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL + AND (operation_id IS NULL OR operation_id != ?)`).get(server, exceptOperationId); +} + /** Only the latest lifecycle operation can describe the current server. */ export function isProvisioning(server: string): boolean { const operation = db().prepare(`SELECT action, state FROM server_deployments diff --git a/src/lib/server/integrations/plutonium.ts b/src/lib/server/integrations/plutonium.ts index 10124b3..5b658ad 100644 --- a/src/lib/server/integrations/plutonium.ts +++ b/src/lib/server/integrations/plutonium.ts @@ -1,6 +1,7 @@ import { db } from '../db/index.js'; import { config } from '../config.js'; import { getServer, listServers } from '../fleet/servers.js'; +import { hasActiveProvisioning } from '../fleet/provisioning.js'; import { latestGameSubmission } from '../fleet/status-evidence.js'; import { rconPlutoniumVersion } from './rcon.js'; import { komodoInspectHealthy } from './komodo.js'; @@ -34,15 +35,21 @@ export function runningVersions(): PlutoniumVersion[] { : revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' }; }); } -export async function probeVersion(server: string): Promise { +export async function probeVersion(server: string, options: { maintenance?: boolean } = {}): Promise { + // Startup verification calls this directly; only maintenance observers yield + // ownership, without replacing provisioning's version evidence with an error. + const suppressed = () => options.maintenance && hasActiveProvisioning(server); + if (suppressed()) return null; const original = getServer(server); if (!original) return null; const epoch = latestGameSubmission(server); const recovering = restartRecoveryPending(server); const health = !recovering && original.desired_state !== 'stopped' ? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false }; + if (suppressed()) return null; let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' } : await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' })); + if (suppressed()) return null; const current = getServer(server); // An older probe must not clear evidence already recorded for a new process // or replacement registration, including replacements with no submission yet. @@ -72,7 +79,7 @@ async function check(): Promise { await checkRelease(); const servers = listServers(); for (let i = 0; i < servers.length; i += 4) { - await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name))); + await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name, { maintenance: true }))); } writeSetting('plutonium_versions_checked_at', nowIso()); } diff --git a/src/lib/server/maintenance/recovery.ts b/src/lib/server/maintenance/recovery.ts index 6e0d850..3344ab3 100644 --- a/src/lib/server/maintenance/recovery.ts +++ b/src/lib/server/maintenance/recovery.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto'; import { db } from '../db/index.js'; import { komodoConfigured } from '../config.js'; import { getServer, listServers } from '../fleet/servers.js'; +import { hasActiveProvisioning } from '../fleet/provisioning.js'; import { komodoInspectHealthy } from '../integrations/komodo.js'; import { rconStatusResult } from '../integrations/rcon.js'; import { telemetryEvidence } from '../fleet/status-evidence.js'; @@ -73,8 +74,17 @@ export function watchdogDetail(id: string): WatchdogDetail | null { const row = incident(id); return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null; } -function hasPending(server: string) { - return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server); +function scanSuppressed(server: string, kind?: 'health' | 'drift'): boolean { + // An explicit retry can reclaim its own earlier attempt's recovery slot. + // Keep other findings and every active lifecycle operation suppressed. + const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r + JOIN server_deployments d ON d.id = r.operation_id AND d.server_deleted_at IS NULL + JOIN watchdog_actions a ON a.operation_id = d.id + JOIN watchdog_incidents i ON i.id = a.incident_id + WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled') + AND i.server = r.server AND i.server_deleted_at IS NULL AND i.state = 'open' AND a.cycle < i.cycle + AND (? IS NULL OR i.kind = ?) LIMIT 1`).get(server, kind ?? null, kind ?? null) as { operation_id: string } | undefined; + return hasActiveProvisioning(server, retained?.operation_id); } function protectedWork(server: string): boolean { return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server) @@ -111,6 +121,9 @@ async function reconcileActions(): Promise { for (const row of rows) { const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1); if (!action) continue; + // Continue observing our own recovery, but do not verify or escalate + // against files and health owned by another lifecycle operation. + if (hasActiveProvisioning(row.server, action.operation_id)) continue; const operation = db().prepare('SELECT state,submitted_at,started_at,stage,error FROM server_deployments WHERE id=?').get(action.operation_id) as { state: string; submitted_at: string | null; started_at: string; stage: string; error: string | null } | undefined; if (!operation) { transition(row.id, 'attention', 'Provisioning history is missing; inspect the server before retrying'); continue; } @@ -118,9 +131,13 @@ async function reconcileActions(): Promise { if (row.kind === 'drift') { try { const plan = await previewDrift([row.server]); - if (incident(row.id)?.cycle !== row.cycle) continue; + if (incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue; if (plan.files.length) { transition(row.id, 'attention', 'Rebuild completed but drift remains; inspect the campaign and server files'); continue; } - } catch (err) { transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`); continue; } + } catch (err) { + if (!hasActiveProvisioning(row.server, action.operation_id)) + transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`); + continue; + } } transition(row.id, 'resolved', action.action === 'restart' ? 'Server recovered after restart' : 'Server recovered after rebuild'); continue; @@ -150,8 +167,9 @@ async function reconcileActions(): Promise { } interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null } -async function observe(server: string): Promise { +async function observe(server: string): Promise { const health = await komodoInspectHealthy(server); + if (scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return null; const state = health.state?.toLowerCase() ?? null; if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state }; if (telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state }; @@ -162,14 +180,14 @@ async function observe(server: string): Promise { async function scanServer(server: string): Promise { const row = getServer(server); if (!row || row.desired_state === 'stopped') return; - if (hasPending(server) || protectedWork(server)) return; + if (scanSuppressed(server) || protectedWork(server)) return; const observation = await observe(server); // Probes can overlap an API request; recheck ownership before queuing anything. - if (hasPending(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return; - if (observation.healthy) { + if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return; + if (!scanSuppressed(server, 'health') && observation.healthy) { const current = active(server, 'health'); if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified'); - } else { + } else if (!scanSuppressed(server, 'health')) { const problem = detect(server, 'health', observation.reason, observation); const startup = ['starting','restarting'].includes(observation.containerState ?? ''); const oldEnough = Date.now() - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS); @@ -181,11 +199,11 @@ async function scanServer(server: string): Promise { })(); if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error); } - // Unknown container health is an observation failure, not proof that a rebuild is safe. - if (!observation.actionable || hasPending(server)) return; } + // Unknown container health is an observation failure, not proof that a rebuild is safe. + if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return; const prepared = await prepareDriftCampaign(server); - if (hasPending(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return; + if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return; if (!prepared.plan.files.length) { const current = active(server, 'drift'); if (current) transition(current.id, 'resolved', 'Server files match the promoted setup'); @@ -225,14 +243,16 @@ async function run(forceScan: boolean): Promise { while (cursor < servers.length) { const server = servers[cursor++]; try { await scanServer(server.name); } - catch (err) { errors.push(`${server.name}: ${(err as Error).message}`); } + catch (err) { + if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`); + } } } await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump)); // Preserve incidents for temporarily absent rebuild instances, but close retired registrations. const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'").all() as WatchdogIncident[]; for (const row of abandoned) { - if (getServer(row.server) || hasPending(row.server)) continue; + if (getServer(row.server) || hasActiveProvisioning(row.server)) continue; const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined; // An absent registry can also mean a failed rebuild. Only a completed // destruction retires that failure without requiring inspection. diff --git a/src/lib/server/maintenance/runtime.ts b/src/lib/server/maintenance/runtime.ts index 0004c81..59ecfdb 100644 --- a/src/lib/server/maintenance/runtime.ts +++ b/src/lib/server/maintenance/runtime.ts @@ -1,5 +1,6 @@ import { db } from '../db/index.js'; import { getServer, listServers } from '../fleet/servers.js'; +import { hasActiveProvisioning } from '../fleet/provisioning.js'; import { requestRestart, cancelProvisioning } from '../provisioning/index.js'; import { nextMaintenanceAt } from '../../plutonium.js'; import type { MaintenanceJob, MaintenanceSchedule, PlutoniumStatus } from '../../plutonium.js'; @@ -37,6 +38,7 @@ export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind'] let added = 0; for (const server of new Set(servers)) { if (getServer(server)?.desired_state === 'stopped') continue; + if (origin !== 'manual' && hasActiveProvisioning(server)) continue; added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin) VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes; const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')").get(server) as { id: number }; @@ -125,6 +127,9 @@ async function runTick(): Promise { async function processJob(job: MaintenanceJob): Promise { const server = getServer(job.server); if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.'); + if (!job.provisioning_id && !job.restarted_at && hasActiveProvisioning(job.server)) { + return hold(job, 'Waiting for another provisioning operation.'); + } const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) }); if (['failed','interrupted','cancelled'].includes(operation.state)) { return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage); @@ -136,7 +141,13 @@ async function processJob(job: MaintenanceJob): Promise { } return hold(job, operation.stage); } - const revision = await probeVersion(job.server); + if (hasActiveProvisioning(job.server, operation.id)) { + return hold(job, 'Waiting for another provisioning operation before verifying the running revision.'); + } + const revision = await probeVersion(job.server, { maintenance: true }); + if (hasActiveProvisioning(job.server, operation.id)) { + return hold(job, 'Waiting for another provisioning operation before verifying the running revision.'); + } if (job.target_revision && (revision === null || revision < job.target_revision)) { return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`); } diff --git a/tests/unit/plutonium.test.ts b/tests/unit/plutonium.test.ts index 0d6b6c5..4d50b3d 100644 --- a/tests/unit/plutonium.test.ts +++ b/tests/unit/plutonium.test.ts @@ -31,6 +31,78 @@ afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAl const api = () => import('../../src/lib/server/fleet/plutonium.js'); describe('Plutonium versions', () => { + it.each([ + ['create', 'queued'], ['rebuild', 'waiting'], ['start', 'running'], + ['restart', 'awaiting-health'], ['stop', 'waiting'], ['destroy', 'running'] + ])('suppresses fleet probes during %s (%s), preserves evidence, and resumes afterwards', async (action, state) => { + const m = await api(); + await m.checkPlutonium(); + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('busy', ?, ?, ?, 'Provisioning', ?, 'test')`).run(servers[0], action, state, new Date().toISOString()); + closeDb(); + vi.mocked(komodoInspectHealthy).mockClear(); + vi.mocked(rconPlutoniumVersion).mockClear(); + vi.setSystemTime(Date.now() + 1_000); + await m.checkPlutonium(); + expect(komodoInspectHealthy).toHaveBeenCalledTimes(1); + expect(komodoInspectHealthy).toHaveBeenCalledWith(servers[1]); + expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1); + expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[1]); + expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved); + db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'busy'").run(); + await m.checkPlutonium(); + expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]); + expect(db().prepare('SELECT checked_at FROM plutonium_versions WHERE server = ?').get(servers[0])) + .toEqual({ checked_at: new Date().toISOString() }); + }); + it('suppresses fleet probes while a failed restart retains recovery capacity', async () => { + const m = await api(); + await m.checkPlutonium(); + const { db } = await import('../../src/lib/server/db/index.js'); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('failed-restart', ?, 'restart', 'failed', 'Timed out', ?, 'test')`).run(servers[0], new Date().toISOString()); + db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)') + .run(servers[0], new Date().toISOString(), 'failed-restart'); + vi.mocked(komodoInspectHealthy).mockClear(); + vi.mocked(rconPlutoniumVersion).mockClear(); + await m.checkPlutonium(); + expect(komodoInspectHealthy).not.toHaveBeenCalledWith(servers[0]); + expect(rconPlutoniumVersion).not.toHaveBeenCalledWith(servers[0]); + expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved); + db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]); + vi.setSystemTime(Date.now() + 1_000); + await m.checkPlutonium(); + expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]); + }); + it.each(['health', 'version'] as const)('discards maintenance observations when provisioning is queued during the %s probe', async phase => { + const m = await api(); + await m.checkPlutonium(); + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); + const queue = () => db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('new-rebuild', ?, 'rebuild', 'waiting', 'Waiting', ?, 'test')`).run(servers[0], new Date().toISOString()); + vi.mocked(rconPlutoniumVersion).mockClear(); + if (phase === 'health') vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => { + queue(); return { ok: true, state: 'running' }; + }); + else vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { + queue(); return { revision: null, error: 'Restarting' }; + }); + vi.setSystemTime(Date.now() + 1_000); + expect(await probeVersion(servers[0], { maintenance: true })).toBeNull(); + if (phase === 'health') expect(rconPlutoniumVersion).not.toHaveBeenCalled(); + expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved); + }); it('coalesces published-release checks without probing the fleet', async () => { const { checkRelease, release } = await import('../../src/lib/server/integrations/plutonium.js'); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); @@ -292,6 +364,88 @@ describe('persistent maintenance', () => { expect(komodoRestart).not.toHaveBeenCalled(); expect(m.plutoniumStatus().jobs[0].detail).toContain('another provisioning operation'); }); + it('waits to attach legacy maintenance while another operation owns the server', async () => { + const m = await api(); await m.checkPlutonium(); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare(`INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)`).run(servers[0], new Date().toISOString()); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString()); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', provisioning_id: null, detail: expect.stringContaining('another provisioning operation') }); + expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 1 }); + db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'").run(); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0].provisioning_id).toBeTruthy(); + expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 2 }); + }); + it.each([false, true])('adopts a submitted legacy restart with retained recovery capacity (linked operation: %s)', async linked => { + const m = await api(); await m.checkPlutonium(); + const { db } = await import('../../src/lib/server/db/index.js'); + const submittedAt = new Date().toISOString(); + db().prepare(`INSERT INTO maintenance_jobs (server, requested_at, restarted_at, state) + VALUES (?, ?, ?, 'restarting')`).run(servers[0], submittedAt, submittedAt); + if (linked) { + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('legacy-restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt); + } + db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)') + .run(servers[0], submittedAt, linked ? 'legacy-restart' : null); + await m.maintenanceTick(); + const job = m.plutoniumStatus().jobs[0]; + expect(job).toMatchObject({ state: 'restarting', operation_state: 'awaiting-health' }); + expect(job.provisioning_id).toBeTruthy(); + if (linked) expect(job.provisioning_id).toBe('legacy-restart'); + expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(servers[0])).toEqual({ operation_id: job.provisioning_id }); + vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); + expect(m.plutoniumStatus().jobs[0].state).toBe('done'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + expect(komodoRestart).not.toHaveBeenCalled(); + }); + it('defers completed maintenance verification while a later operation owns the server, then resumes', async () => { + const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const { db } = await import('../../src/lib/server/db/index.js'); + const job = m.plutoniumStatus().jobs[0]; + db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(job.provisioning_id); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('later-restart', ?, 'restart', 'waiting', 'Waiting to restart', ?, 'test')`).run(servers[0], new Date().toISOString()); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(komodoInspectHealthy).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear(); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') }); + expect(komodoInspectHealthy).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled(); + db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-restart'").run(); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0].state).toBe('done'); + expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]); + }); + it('does not fail maintenance verification when another operation starts during its version probe', async () => { + const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(m.plutoniumStatus().jobs[0].provisioning_id); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('later-rebuild', ?, 'rebuild', 'running', 'Rebuilding', ?, 'test')`).run(servers[0], new Date().toISOString()); + return { revision: null, error: 'Restarting' }; + }); + const before = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') }); + expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(before); + db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-rebuild'").run(); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0].state).toBe('done'); + }); + it('mirrors its own failed operation even while another operation owns the server', async () => { + const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("UPDATE server_deployments SET state = 'failed', error = 'Restart failed' WHERE id = ?").run(m.plutoniumStatus().jobs[0].provisioning_id); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('later-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString()); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', detail: 'Restart failed' }); + }); it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => { const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); const { db } = await import('../../src/lib/server/db/index.js'); @@ -361,6 +515,58 @@ describe('Maintenance policies and runtime updates', () => { expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[0], kind: 'update', origin: 'automatic', target_revision: 5354 })]); expect(m.plutoniumStatus().servers.find(s => s.server === servers[1])?.update_status).toBe('unknown'); }); + it.each(['queued', 'waiting', 'running', 'awaiting-health'])('skips automatic updates during %s provisioning and resumes after completion', async state => { + const m = await outdated(); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('existing-restart', ?, 'restart', ?, 'Restarting', ?, 'test')`).run(servers[0], state, new Date().toISOString()); + const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'update', origin: 'automatic' })]); + expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server = ?').get(servers[0])).toEqual({ count: 1 }); + db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-restart'").run(); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toHaveLength(2); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'automatic' }); + }); + it('skips a busy server for the scheduled occurrence while other servers proceed', async () => { + const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC'); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString()); + vi.setSystemTime(new Date('2026-09-21T06:00:00Z')); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'schedule' })]); + expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-22T06:00:00.000Z'); + db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'").run(); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toHaveLength(1); + vi.setSystemTime(new Date('2026-09-22T06:00:00Z')); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toHaveLength(2); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'schedule' }); + }); + it('skips automatic and scheduled requests while a failed operation retains recovery capacity', async () => { + const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC'); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('failed-restart', ?, 'restart', 'failed', 'Restart failed', ?, 'test')`).run(servers[0], new Date().toISOString()); + db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, 'failed-restart')`) + .run(servers[0], new Date().toISOString()); + expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(0); + vi.setSystemTime(new Date('2026-09-21T06:00:00Z')); await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'schedule' })]); + db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]); + expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(1); + }); + it('preserves explicit update requests while another operation owns the server', async () => { + const m = await outdated(); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString()); + expect(m.queueUpdates([servers[0]])).toBe(1); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'manual' }); + }); it('holds queued automatic updates when their policy is disabled, but finishes submitted recovery while paused', async () => { const m = await outdated(); const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true); diff --git a/tests/unit/watchdog.test.ts b/tests/unit/watchdog.test.ts index fb26a4c..bf1f1be 100644 --- a/tests/unit/watchdog.test.ts +++ b/tests/unit/watchdog.test.ts @@ -194,6 +194,50 @@ describe('Watchdog detection and ownership', () => { expect(incidents()).toEqual([]); expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]); }); + + it.each(['queued', 'waiting', 'running', 'awaiting-health', 'retained recovery'])( + 'suppresses health and drift throughout %s provisioning, then resumes', async state => { + const retained = state === 'retained recovery'; + const now = new Date().toISOString(); + database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('manual-operation', ?, ?, ?, 'Operator work', ?, 'manual')`) + .run(server, retained ? 'restart' : 'rebuild', retained ? 'failed' : state, now); + if (retained) database.db().prepare(`INSERT INTO restart_rollout (server, operation_id, requested_at) + VALUES (?, 'manual-operation', ?)`).run(server, now); + await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected'); + down(); + for (let n = 0; n < 4; n++) { + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + } + expect(incidents()).toEqual([]); + expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled(); + expect(rcon.rconStatusResult).not.toHaveBeenCalled(); + expect(operations()).toHaveLength(1); + + database.db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'manual-operation'").run(); + if (retained) database.db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), server); + await watchdog.watchdogTick(true); + expect(incidents()).toEqual(expect.arrayContaining([ + expect.objectContaining({ kind: 'health', state: 'open', failures: 1 }), + expect.objectContaining({ kind: 'drift', state: 'recovering' }) + ])); + expect(operations()).toHaveLength(2); + expect(operations()[1]).toMatchObject({ action: 'rebuild', state: 'waiting' }); + } + ); + + it('stops probing when provisioning takes ownership during the container inspection', async () => { + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => { + expect(requestLifecycle(server, 'restart').ok).toBe(true); + return { ok: true, state: 'running' }; + }); + await watchdog.watchdogTick(true); + expect(incidents()).toEqual([]); + expect(rcon.rconStatusResult).not.toHaveBeenCalled(); + expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]); + }); }); describe('Watchdog recovery persistence', () => { @@ -254,6 +298,44 @@ describe('Watchdog recovery persistence', () => { expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']); }); + it('reclaims a failed attempt recovery slot only after an explicit incident retry', async () => { + const row = await sustainedFailure(); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + advance(watchdog.WATCHDOG_RECOVERY_MS); + await watchdog.watchdogTick(true); + vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' }); + await processProvisioning(); + await watchdog.watchdogTick(true); + const failed = operations()[1]; + expect(failed).toMatchObject({ action: 'rebuild', state: 'failed' }); + expect(onlyIncident()).toMatchObject({ id: row.id, state: 'attention', cycle: 1 }); + const slot = () => database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server); + expect(slot()).toEqual({ operation_id: failed.id, healthy_at: null }); + + await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected'); + vi.mocked(komodo.komodoInspectHealthy).mockClear(); + advance(watchdog.WATCHDOG_RECOVERY_MS); + await watchdog.watchdogTick(true); + expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled(); + expect(operations()).toHaveLength(2); + + expect(watchdog.updateWatchdogIncident(row.id, 'retry')).toEqual({ ok: true }); + await watchdog.watchdogTick(true); + expect(operations()).toHaveLength(2); + advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + const retry = operations()[2]; + expect(retry).toMatchObject({ action: 'restart', state: 'waiting' }); + expect(onlyIncident()).toMatchObject({ id: row.id, state: 'recovering', cycle: 2 }); + expect(slot()).toEqual({ operation_id: retry.id, healthy_at: null }); + expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(3); + + await processProvisioning(); + expect(komodo.komodoRestart).toHaveBeenCalledTimes(2); + expect(operations()[2].state).toBe('awaiting-health'); + }); + it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => { down(); await watchdog.watchdogTick(true);