josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: make provisioning own instance lifecycle operations

Browse Source
This commit is contained in:
josh
2026-09-27 14:45:00 -04:00
parent cfed584614
commit df265d1900
44 changed files with 1791 additions and 1609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/plutonium.test.ts
+39 -16
View File
@@ -1,3 +1,4 @@
import { maintenanceTick } from '../helpers/provisioning.js';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { maintenanceTimeInZone, nextMaintenanceAt, parsePlutoniumManifest, parsePlutoniumRevision, PLUTONIUM_MANIFEST_URL } from '../../src/lib/plutonium.js';
@@ -100,12 +101,31 @@ describe('maintenance timezones', () => {
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-03-07T11:00:00.000Z');
vi.setSystemTime(new Date('2026-03-07T11:00:00Z'));
await m.maintenanceTick();
await maintenanceTick();
expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-03-08T10:00:00.000Z' });
});
});
describe('persistent maintenance', () => {
it('submits lifecycle work without executing it and rejects cancellation after Provisioning submits', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
await m.maintenanceTick();
expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
const job = m.plutoniumStatus().jobs[0];
expect(job.provisioning_id).toBeTruthy();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
// The source has not mirrored submission yet; cancellation must still
// consult the authoritative operation, not its stale queued state.
expect(m.plutoniumStatus().jobs[0].state).toBe('queued');
expect(m.cancelMaintenance(job.id)).toBe(false);
expect(komodoRestart).toHaveBeenCalledTimes(1);
vi.setSystemTime(Date.now() + 15_000); await processProvisioning();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
});
it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => {
const m = await api(); await m.checkPlutonium();
expect(m.queueMaintenance([servers[0]])).toBe(1);
@@ -113,31 +133,31 @@ describe('persistent maintenance', () => {
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null);
await m.maintenanceTick(); await m.maintenanceTick();
await maintenanceTick(); await maintenanceTick();
expect(komodoRestart).not.toHaveBeenCalled();
await Promise.all([m.maintenanceTick(), m.maintenanceTick()]);
await Promise.all([maintenanceTick(), maintenanceTick()]);
expect(komodoRestart).toHaveBeenCalledTimes(1);
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
await m.maintenanceTick();
await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
vi.setSystemTime(Date.now() + 15_000);
await m.maintenanceTick();
await maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' });
});
it('shares rolling capacity across servers and survives closing the database', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers);
await m.maintenanceTick();
await maintenanceTick();
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']);
vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick();
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']);
});
it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
await m.maintenanceTick();
await maintenanceTick();
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick(); await m.maintenanceTick();
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 });
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).toHaveBeenCalledTimes(1);
@@ -146,7 +166,7 @@ describe('persistent maintenance', () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' });
await m.maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick();
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
expect(komodoRestart).toHaveBeenCalledTimes(1);
});
@@ -156,7 +176,7 @@ describe('persistent maintenance', () => {
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0;
});
await m.maintenanceTick();
await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
@@ -166,19 +186,22 @@ describe('persistent maintenance', () => {
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString());
await m.maintenanceTick();
await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
expect(m.plutoniumStatus().jobs[0].detail).toContain('deployment or beta');
expect(m.plutoniumStatus().jobs[0].detail).toContain('another provisioning operation');
});
it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => {
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run();
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
expect(komodoRestart).not.toHaveBeenCalled();
});
it('validates API requests without changing the schedule or queue', async () => {
const { POST } = await import('../../src/routes/api/plutonium/+server.js');
@@ -203,7 +226,7 @@ describe('persistent maintenance', () => {
expect(m.plutoniumStatus().schedule.enabled).toBe(false);
m.saveMaintenanceSchedule(true, '06:00', 'UTC');
vi.setSystemTime(new Date('2026-09-24T07:00:00Z'));
await m.maintenanceTick(); await m.maintenanceTick();
await maintenanceTick(); await maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(2);
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
m.saveMaintenanceSchedule(false, '06:00', 'UTC');