josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Suppress maintenance watchers during provisioning

Browse Source 
Keep maintenance observations and automatic requests clear of active lifecycle operations and retained recovery capacity while preserving owned recovery and explicit retries.

Fixes #55
This commit is contained in:
josh
2026-10-01 11:54:22 -04:00
parent a3cd3ee585
commit 0cb48c435e
8 changed files with 366 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/plutonium.test.ts
+206
View File
@@ -31,6 +31,78 @@ afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAl
const api = () => import('../../src/lib/server/fleet/plutonium.js');
describe('Plutonium versions', () => {
it.each([
['create', 'queued'], ['rebuild', 'waiting'], ['start', 'running'],
['restart', 'awaiting-health'], ['stop', 'waiting'], ['destroy', 'running']
])('suppresses fleet probes during %s (%s), preserves evidence, and resumes afterwards', async (action, state) => {
const m = await api();
await m.checkPlutonium();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('busy', ?, ?, ?, 'Provisioning', ?, 'test')`).run(servers[0], action, state, new Date().toISOString());
closeDb();
vi.mocked(komodoInspectHealthy).mockClear();
vi.mocked(rconPlutoniumVersion).mockClear();
vi.setSystemTime(Date.now() + 1_000);
await m.checkPlutonium();
expect(komodoInspectHealthy).toHaveBeenCalledTimes(1);
expect(komodoInspectHealthy).toHaveBeenCalledWith(servers[1]);
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1);
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[1]);
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'busy'").run();
await m.checkPlutonium();
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]);
expect(db().prepare('SELECT checked_at FROM plutonium_versions WHERE server = ?').get(servers[0]))
.toEqual({ checked_at: new Date().toISOString() });
});
it('suppresses fleet probes while a failed restart retains recovery capacity', async () => {
const m = await api();
await m.checkPlutonium();
const { db } = await import('../../src/lib/server/db/index.js');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('failed-restart', ?, 'restart', 'failed', 'Timed out', ?, 'test')`).run(servers[0], new Date().toISOString());
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
.run(servers[0], new Date().toISOString(), 'failed-restart');
vi.mocked(komodoInspectHealthy).mockClear();
vi.mocked(rconPlutoniumVersion).mockClear();
await m.checkPlutonium();
expect(komodoInspectHealthy).not.toHaveBeenCalledWith(servers[0]);
expect(rconPlutoniumVersion).not.toHaveBeenCalledWith(servers[0]);
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]);
vi.setSystemTime(Date.now() + 1_000);
await m.checkPlutonium();
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]);
});
it.each(['health', 'version'] as const)('discards maintenance observations when provisioning is queued during the %s probe', async phase => {
const m = await api();
await m.checkPlutonium();
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
const queue = () => db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('new-rebuild', ?, 'rebuild', 'waiting', 'Waiting', ?, 'test')`).run(servers[0], new Date().toISOString());
vi.mocked(rconPlutoniumVersion).mockClear();
if (phase === 'health') vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => {
queue(); return { ok: true, state: 'running' };
});
else vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
queue(); return { revision: null, error: 'Restarting' };
});
vi.setSystemTime(Date.now() + 1_000);
expect(await probeVersion(servers[0], { maintenance: true })).toBeNull();
if (phase === 'health') expect(rconPlutoniumVersion).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
});
it('coalesces published-release checks without probing the fleet', async () => {
const { checkRelease, release } = await import('../../src/lib/server/integrations/plutonium.js');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
@@ -292,6 +364,88 @@ describe('persistent maintenance', () => {
expect(komodoRestart).not.toHaveBeenCalled();
expect(m.plutoniumStatus().jobs[0].detail).toContain('another provisioning operation');
});
it('waits to attach legacy maintenance while another operation owns the server', async () => {
const m = await api(); await m.checkPlutonium();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)`).run(servers[0], new Date().toISOString());
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString());
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', provisioning_id: null, detail: expect.stringContaining('another provisioning operation') });
expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 1 });
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'").run();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0].provisioning_id).toBeTruthy();
expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 2 });
});
it.each([false, true])('adopts a submitted legacy restart with retained recovery capacity (linked operation: %s)', async linked => {
const m = await api(); await m.checkPlutonium();
const { db } = await import('../../src/lib/server/db/index.js');
const submittedAt = new Date().toISOString();
db().prepare(`INSERT INTO maintenance_jobs (server, requested_at, restarted_at, state)
VALUES (?, ?, ?, 'restarting')`).run(servers[0], submittedAt, submittedAt);
if (linked) {
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('legacy-restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
}
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
.run(servers[0], submittedAt, linked ? 'legacy-restart' : null);
await m.maintenanceTick();
const job = m.plutoniumStatus().jobs[0];
expect(job).toMatchObject({ state: 'restarting', operation_state: 'awaiting-health' });
expect(job.provisioning_id).toBeTruthy();
if (linked) expect(job.provisioning_id).toBe('legacy-restart');
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(servers[0])).toEqual({ operation_id: job.provisioning_id });
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
});
it('defers completed maintenance verification while a later operation owns the server, then resumes', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
const job = m.plutoniumStatus().jobs[0];
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(job.provisioning_id);
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('later-restart', ?, 'restart', 'waiting', 'Waiting to restart', ?, 'test')`).run(servers[0], new Date().toISOString());
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(komodoInspectHealthy).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') });
expect(komodoInspectHealthy).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled();
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-restart'").run();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]);
});
it('does not fail maintenance verification when another operation starts during its version probe', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(m.plutoniumStatus().jobs[0].provisioning_id);
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('later-rebuild', ?, 'rebuild', 'running', 'Rebuilding', ?, 'test')`).run(servers[0], new Date().toISOString());
return { revision: null, error: 'Restarting' };
});
const before = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') });
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(before);
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-rebuild'").run();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
});
it('mirrors its own failed operation even while another operation owns the server', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE server_deployments SET state = 'failed', error = 'Restart failed' WHERE id = ?").run(m.plutoniumStatus().jobs[0].provisioning_id);
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('later-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString());
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', detail: 'Restart failed' });
});
it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
@@ -361,6 +515,58 @@ describe('Maintenance policies and runtime updates', () => {
expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[0], kind: 'update', origin: 'automatic', target_revision: 5354 })]);
expect(m.plutoniumStatus().servers.find(s => s.server === servers[1])?.update_status).toBe('unknown');
});
it.each(['queued', 'waiting', 'running', 'awaiting-health'])('skips automatic updates during %s provisioning and resumes after completion', async state => {
const m = await outdated();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('existing-restart', ?, 'restart', ?, 'Restarting', ?, 'test')`).run(servers[0], state, new Date().toISOString());
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'update', origin: 'automatic' })]);
expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server = ?').get(servers[0])).toEqual({ count: 1 });
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-restart'").run();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(2);
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'automatic' });
});
it('skips a busy server for the scheduled occurrence while other servers proceed', async () => {
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC');
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString());
vi.setSystemTime(new Date('2026-09-21T06:00:00Z'));
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'schedule' })]);
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-22T06:00:00.000Z');
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'").run();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(1);
vi.setSystemTime(new Date('2026-09-22T06:00:00Z'));
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(2);
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'schedule' });
});
it('skips automatic and scheduled requests while a failed operation retains recovery capacity', async () => {
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC');
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('failed-restart', ?, 'restart', 'failed', 'Restart failed', ?, 'test')`).run(servers[0], new Date().toISOString());
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, 'failed-restart')`)
.run(servers[0], new Date().toISOString());
expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(0);
vi.setSystemTime(new Date('2026-09-21T06:00:00Z')); await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'schedule' })]);
db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]);
expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(1);
});
it('preserves explicit update requests while another operation owns the server', async () => {
const m = await outdated();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString());
expect(m.queueUpdates([servers[0]])).toBe(1);
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'manual' });
});
it('holds queued automatic updates when their policy is disabled, but finishes submitted recovery while paused', async () => {
const m = await outdated();
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
tests/unit/watchdog.test.ts
+82
View File
@@ -194,6 +194,50 @@ describe('Watchdog detection and ownership', () => {
expect(incidents()).toEqual([]);
expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
});
it.each(['queued', 'waiting', 'running', 'awaiting-health', 'retained recovery'])(
'suppresses health and drift throughout %s provisioning, then resumes', async state => {
const retained = state === 'retained recovery';
const now = new Date().toISOString();
database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('manual-operation', ?, ?, ?, 'Operator work', ?, 'manual')`)
.run(server, retained ? 'restart' : 'rebuild', retained ? 'failed' : state, now);
if (retained) database.db().prepare(`INSERT INTO restart_rollout (server, operation_id, requested_at)
VALUES (?, 'manual-operation', ?)`).run(server, now);
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
down();
for (let n = 0; n < 4; n++) {
await watchdog.watchdogTick(true);
advance(watchdog.WATCHDOG_SCAN_MS);
}
expect(incidents()).toEqual([]);
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
expect(operations()).toHaveLength(1);
database.db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'manual-operation'").run();
if (retained) database.db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), server);
await watchdog.watchdogTick(true);
expect(incidents()).toEqual(expect.arrayContaining([
expect.objectContaining({ kind: 'health', state: 'open', failures: 1 }),
expect.objectContaining({ kind: 'drift', state: 'recovering' })
]));
expect(operations()).toHaveLength(2);
expect(operations()[1]).toMatchObject({ action: 'rebuild', state: 'waiting' });
}
);
it('stops probing when provisioning takes ownership during the container inspection', async () => {
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => {
expect(requestLifecycle(server, 'restart').ok).toBe(true);
return { ok: true, state: 'running' };
});
await watchdog.watchdogTick(true);
expect(incidents()).toEqual([]);
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
});
});
describe('Watchdog recovery persistence', () => {
@@ -254,6 +298,44 @@ describe('Watchdog recovery persistence', () => {
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
});
it('reclaims a failed attempt recovery slot only after an explicit incident retry', async () => {
const row = await sustainedFailure();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
advance(watchdog.WATCHDOG_RECOVERY_MS);
await watchdog.watchdogTick(true);
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' });
await processProvisioning();
await watchdog.watchdogTick(true);
const failed = operations()[1];
expect(failed).toMatchObject({ action: 'rebuild', state: 'failed' });
expect(onlyIncident()).toMatchObject({ id: row.id, state: 'attention', cycle: 1 });
const slot = () => database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server);
expect(slot()).toEqual({ operation_id: failed.id, healthy_at: null });
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
vi.mocked(komodo.komodoInspectHealthy).mockClear();
advance(watchdog.WATCHDOG_RECOVERY_MS);
await watchdog.watchdogTick(true);
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
expect(operations()).toHaveLength(2);
expect(watchdog.updateWatchdogIncident(row.id, 'retry')).toEqual({ ok: true });
await watchdog.watchdogTick(true);
expect(operations()).toHaveLength(2);
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
const retry = operations()[2];
expect(retry).toMatchObject({ action: 'restart', state: 'waiting' });
expect(onlyIncident()).toMatchObject({ id: row.id, state: 'recovering', cycle: 2 });
expect(slot()).toEqual({ operation_id: retry.id, healthy_at: null });
expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(3);
await processProvisioning();
expect(komodo.komodoRestart).toHaveBeenCalledTimes(2);
expect(operations()[2].state).toBe('awaiting-health');
});
it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
down();
await watchdog.watchdogTick(true);