josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Komodo backend in favor of native hosts

Browse Source
This commit is contained in:
josh
2026-10-05 18:07:07 -04:00
parent 701066aaf8
commit 4b0f341d64
116 changed files with 557 additions and 8132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/plutonium.test.ts
+22 -236
View File
@@ -3,7 +3,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { maintenanceTimeInZone, nextMaintenanceAt, parsePlutoniumManifest, parsePlutoniumRevision, PLUTONIUM_MANIFEST_URL } from '../../src/lib/plutonium.js';
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() }));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
let fixture: Fixture;
@@ -13,15 +12,15 @@ beforeEach(async () => {
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(new Date('2026-09-21T05:59:00Z'));
fixture = await newFixture(servers);
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test');
vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
vi.stubGlobal('fetch', vi.fn().mockImplementation(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
const k = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(k.komodoRestart).mockResolvedValue({ ok: true });
vi.mocked(k.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth');
const k = await import('../../src/lib/server/backends/native.js');
vi.mocked(k.nativeHealth).mockResolvedValue({ ok: true, state: 'running' });
const r = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0);
vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
@@ -38,18 +37,18 @@ describe('Plutonium versions', () => {
const m = await api();
await m.checkPlutonium();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('busy', ?, ?, ?, 'Provisioning', ?, 'test')`).run(servers[0], action, state, new Date().toISOString());
closeDb();
vi.mocked(komodoInspectHealthy).mockClear();
vi.mocked(nativeHealth).mockClear();
vi.mocked(rconPlutoniumVersion).mockClear();
vi.setSystemTime(Date.now() + 1_000);
await m.checkPlutonium();
expect(komodoInspectHealthy).toHaveBeenCalledTimes(1);
expect(komodoInspectHealthy).toHaveBeenCalledWith(servers[1]);
expect(nativeHealth).toHaveBeenCalledTimes(1);
expect(nativeHealth).toHaveBeenCalledWith(servers[1]);
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1);
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[1]);
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
@@ -63,17 +62,17 @@ describe('Plutonium versions', () => {
const m = await api();
await m.checkPlutonium();
const { db } = await import('../../src/lib/server/db/index.js');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('failed-restart', ?, 'restart', 'failed', 'Timed out', ?, 'test')`).run(servers[0], new Date().toISOString());
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
.run(servers[0], new Date().toISOString(), 'failed-restart');
vi.mocked(komodoInspectHealthy).mockClear();
vi.mocked(nativeHealth).mockClear();
vi.mocked(rconPlutoniumVersion).mockClear();
await m.checkPlutonium();
expect(komodoInspectHealthy).not.toHaveBeenCalledWith(servers[0]);
expect(nativeHealth).not.toHaveBeenCalledWith(servers[0]);
expect(rconPlutoniumVersion).not.toHaveBeenCalledWith(servers[0]);
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]);
@@ -86,13 +85,13 @@ describe('Plutonium versions', () => {
await m.checkPlutonium();
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
const queue = () => db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('new-rebuild', ?, 'rebuild', 'waiting', 'Waiting', ?, 'test')`).run(servers[0], new Date().toISOString());
vi.mocked(rconPlutoniumVersion).mockClear();
if (phase === 'health') vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => {
if (phase === 'health') vi.mocked(nativeHealth).mockImplementationOnce(async () => {
queue(); return { ok: true, state: 'running' };
});
else vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
@@ -105,7 +104,7 @@ describe('Plutonium versions', () => {
});
it('coalesces published-release checks without probing the fleet', async () => {
const { checkRelease, release } = await import('../../src/lib/server/integrations/plutonium.js');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const first = checkRelease();
const second = checkRelease();
@@ -113,7 +112,7 @@ describe('Plutonium versions', () => {
await Promise.all([first, second]);
expect(fetch).toHaveBeenCalledTimes(1);
expect(release()).toMatchObject({ revision: 5354, error: null });
expect(komodoInspectHealthy).not.toHaveBeenCalled();
expect(nativeHealth).not.toHaveBeenCalled();
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
await checkRelease();
expect(fetch).toHaveBeenCalledTimes(2);
@@ -266,7 +265,7 @@ describe('Plutonium versions', () => {
// Both registrations have no lifecycle submission, so only their
// registry identity distinguishes the old and replacement instances.
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
db().prepare('INSERT INTO servers (name, komodo_name, created_at) VALUES (?, ?, ?)').run(servers[0], servers[0], now);
db().prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)').run(servers[0], now);
}
vi.setSystemTime(Date.now() + 1);
expect(await probeVersion(servers[0])).toBe(5354);
@@ -314,103 +313,9 @@ describe('maintenance timezones', () => {
expect(maintenanceTimeInZone(schedule, 'America/New_York')).toBe('02:00');
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-21T06:00:00.000Z');
});
it('persists the zone and advances in local calendar days across DST', async () => {
vi.setSystemTime(new Date('2026-03-07T10:59:00Z'));
const m = await api(); await m.checkPlutonium();
m.saveMaintenanceSchedule(true, '06:00', 'America/New_York');
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-03-07T11:00:00.000Z');
vi.setSystemTime(new Date('2026-03-07T11:00:00Z'));
await maintenanceTick();
expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-03-08T10:00:00.000Z' });
});
});
describe('persistent maintenance', () => {
it('submits lifecycle work without executing it and rejects cancellation after Provisioning submits', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
await m.maintenanceTick();
expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
const job = m.plutoniumStatus().jobs[0];
expect(job.provisioning_id).toBeTruthy();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
// The source has not mirrored submission yet; cancellation must still
// consult the authoritative operation, not its stale queued state.
expect(m.plutoniumStatus().jobs[0].state).toBe('queued');
expect(m.cancelMaintenance(job.id)).toBe(false);
expect(komodoRestart).toHaveBeenCalledTimes(1);
vi.setSystemTime(Date.now() + 15_000); await processProvisioning();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
});
it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => {
const m = await api(); await m.checkPlutonium();
expect(m.queueMaintenance([servers[0]])).toBe(1);
expect(m.queueMaintenance([servers[0]])).toBe(0);
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null);
await maintenanceTick(); await maintenanceTick();
expect(komodoRestart).not.toHaveBeenCalled();
await Promise.all([maintenanceTick(), maintenanceTick()]);
expect(komodoRestart).toHaveBeenCalledTimes(1);
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
vi.setSystemTime(Date.now() + 15_000);
await maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' });
});
it('shares rolling capacity across servers and survives closing the database', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers);
await maintenanceTick();
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']);
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']);
});
it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
await maintenanceTick();
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 });
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).toHaveBeenCalledTimes(1);
});
it('does not retry an ambiguous restart timeout', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' });
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
expect(komodoRestart).toHaveBeenCalledTimes(1);
});
it('honors cancellation arriving during the player check', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0;
});
await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
});
it('holds maintenance while lifecycle work is queued', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString());
await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
expect(m.plutoniumStatus().jobs[0].detail).toContain('another provisioning operation');
});
it('waits to attach legacy maintenance while another operation owns the server', async () => {
const m = await api(); await m.checkPlutonium();
const { db } = await import('../../src/lib/server/db/index.js');
@@ -425,29 +330,6 @@ describe('persistent maintenance', () => {
expect(m.plutoniumStatus().jobs[0].provisioning_id).toBeTruthy();
expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 2 });
});
it.each([false, true])('adopts a submitted legacy restart with retained recovery capacity (linked operation: %s)', async linked => {
const m = await api(); await m.checkPlutonium();
const { db } = await import('../../src/lib/server/db/index.js');
const submittedAt = new Date().toISOString();
db().prepare(`INSERT INTO maintenance_jobs (server, requested_at, restarted_at, state)
VALUES (?, ?, ?, 'restarting')`).run(servers[0], submittedAt, submittedAt);
if (linked) {
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('legacy-restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
}
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
.run(servers[0], submittedAt, linked ? 'legacy-restart' : null);
await m.maintenanceTick();
const job = m.plutoniumStatus().jobs[0];
expect(job).toMatchObject({ state: 'restarting', operation_state: 'awaiting-health' });
expect(job.provisioning_id).toBeTruthy();
if (linked) expect(job.provisioning_id).toBe('legacy-restart');
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(servers[0])).toEqual({ operation_id: job.provisioning_id });
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
});
it('defers completed maintenance verification while a later operation owns the server, then resumes', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
@@ -455,12 +337,12 @@ describe('persistent maintenance', () => {
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(job.provisioning_id);
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('later-restart', ?, 'restart', 'waiting', 'Waiting to restart', ?, 'test')`).run(servers[0], new Date().toISOString());
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(komodoInspectHealthy).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear();
vi.mocked(nativeHealth).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') });
expect(komodoInspectHealthy).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled();
expect(nativeHealth).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled();
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-restart'").run();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
@@ -493,18 +375,6 @@ describe('persistent maintenance', () => {
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', detail: 'Restart failed' });
});
it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run();
await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
expect(komodoRestart).not.toHaveBeenCalled();
});
it('validates API requests without changing the schedule or queue', async () => {
const { POST } = await import('../../src/routes/api/plutonium/+server.js');
const post = (body: unknown) => POST({ request: new Request('http://localhost/api/plutonium', {
@@ -521,19 +391,6 @@ describe('persistent maintenance', () => {
expect(m.plutoniumStatus().schedule.enabled).toBe(true);
expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-09-21T10:00:00.000Z' });
});
it('runs a daily UTC schedule once and coalesces missed days', async () => {
expect(nextMaintenanceAt('06:00', new Date('2026-09-21T06:00:00Z'))).toBe('2026-09-22T06:00:00.000Z');
expect(() => nextMaintenanceAt('24:00')).toThrow();
const m = await api(); await m.checkPlutonium();
expect(m.plutoniumStatus().schedule.enabled).toBe(false);
m.saveMaintenanceSchedule(true, '06:00', 'UTC');
vi.setSystemTime(new Date('2026-09-24T07:00:00Z'));
await maintenanceTick(); await maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(2);
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
m.saveMaintenanceSchedule(false, '06:00', 'UTC');
expect(m.plutoniumStatus().schedule.nextAt).toBeNull();
});
});
describe('Maintenance policies and runtime updates', () => {
@@ -614,64 +471,6 @@ describe('Maintenance policies and runtime updates', () => {
expect(m.queueUpdates([servers[0]])).toBe(1);
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'manual' });
});
it('holds queued automatic updates when their policy is disabled, but finishes submitted recovery while paused', async () => {
const m = await outdated();
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
m.queueUpdates([servers[0]], true); p.setMaintenancePolicy('updates', false);
await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
p.setMaintenancePolicy('updates', true); p.pauseMaintenance(true);
await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
p.pauseMaintenance(false); await processProvisioning(); expect(komodoRestart).toHaveBeenCalledTimes(1);
p.pauseMaintenance(true); vi.setSystemTime(Date.now() + 15_000);
await processProvisioning(); await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', operation_state: 'done' });
expect(komodoRestart).toHaveBeenCalledTimes(1);
});
it('latches failed update verification across restarts and retention, and supports an explicit retry', async () => {
const m = await outdated();
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[1]);
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
const failed = m.plutoniumStatus().jobs[0];
expect(failed).toMatchObject({ kind: 'update', state: 'failed', operation_state: 'done' });
closeDb(); vi.setSystemTime(Date.now() + 31 * 86_400_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(1);
expect(m.retryMaintenance(failed.id)).toBe(1);
expect(m.plutoniumStatus().jobs).toHaveLength(2);
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', origin: 'manual' });
expect(db().prepare("SELECT 1 FROM server_deployments WHERE action = 'rebuild'").get()).toBeUndefined();
});
it('does not restart intentionally stopped servers, including jobs queued before the intent changed', async () => {
const m = await outdated(); m.queueUpdates([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE servers SET desired_state = 'stopped'").run();
await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
expect(m.queueUpdates()).toBe(0);
expect(m.plutoniumStatus().servers.every(s => s.update_status === 'stopped')).toBe(true);
});
it('separates scheduled restart success from revision verification and coalesces paused runs', async () => {
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC');
const p = await import('../../src/lib/server/maintenance/policies.js'); p.pauseMaintenance(true);
vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(0);
p.pauseMaintenance(false); await maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(2);
expect(m.plutoniumStatus().jobs.every(j => j.kind === 'schedule' && j.target_revision === null)).toBe(true);
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Unknown reply' });
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs.find(j => j.server === servers[0])?.state).toBe('done');
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
});
it('does not treat stale, pre-restart, or unhealthy observations as current runtime evidence', async () => {
const m = await outdated();
const { db } = await import('../../src/lib/server/db/index.js');
@@ -679,25 +478,12 @@ describe('Maintenance policies and runtime updates', () => {
expect(m.plutoniumStatus().servers[0].update_status).toBe('unknown');
vi.setSystemTime(Date.now() + 3_600_000);
expect(() => m.queueUpdates()).toThrow('published revision');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockClear();
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
vi.mocked(nativeHealth).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
await m.checkPlutonium();
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
expect(m.plutoniumStatus().servers.every(s => s.update_status === 'unknown')).toBe(true);
});
it('adopts existing maintenance jobs without changing their IDs, schedules, or submitted operations', async () => {
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'America/New_York');
m.queueMaintenance([servers[0]]); await maintenanceTick();
const before = m.plutoniumStatus();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().exec('ALTER TABLE maintenance_jobs DROP COLUMN kind; ALTER TABLE maintenance_jobs DROP COLUMN origin;'); closeDb();
const after = m.plutoniumStatus();
expect(after.schedule).toEqual(before.schedule);
expect(after.jobs[0]).toMatchObject({ id: before.jobs[0].id, provisioning_id: before.jobs[0].provisioning_id, kind: 'legacy', origin: 'manual' });
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).toHaveBeenCalledTimes(1);
});
});