861 lines
41 KiB
TypeScript
861 lines
41 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import {
|
|
maintenanceTimeInZone,
|
|
nextMaintenanceAt,
|
|
parsePlutoniumManifest,
|
|
parsePlutoniumRevision,
|
|
PLUTONIUM_MANIFEST_URL
|
|
} from '../../src/lib/plutonium.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconGetPlayerCount: vi.fn(),
|
|
rconStatusResult: vi.fn(),
|
|
rconPlutoniumVersion: vi.fn()
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
const servers = ['t5_zombie_theater_1', 't6_zombie_town_1'];
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.resetAllMocks();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(new Date('2026-09-21T05:59:00Z'));
|
|
fixture = await newFixture(servers);
|
|
|
|
vi.stubEnv('RCON_PASSWORD', 'test');
|
|
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
|
vi.stubGlobal(
|
|
'fetch',
|
|
vi
|
|
.fn()
|
|
.mockImplementation(async () =>
|
|
Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })
|
|
)
|
|
);
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth');
|
|
const k = await import('../../src/lib/server/backends/native.js');
|
|
|
|
vi.mocked(k.nativeHealth).mockResolvedValue({ ok: true, state: 'running' });
|
|
const r = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0);
|
|
vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
|
vi.mocked(r.rconStatusResult).mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'test', port: 28960 },
|
|
status: { players: 0, map: null, gametype: null, roster: [] }
|
|
});
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
vi.unstubAllEnvs();
|
|
vi.unstubAllGlobals();
|
|
vi.useRealTimers();
|
|
});
|
|
const api = () => import('../../src/lib/server/fleet/plutonium.js');
|
|
|
|
describe('Plutonium versions', () => {
|
|
it.each([
|
|
['create', 'queued'],
|
|
['rebuild', 'waiting'],
|
|
['start', 'running'],
|
|
['restart', 'awaiting-health'],
|
|
['stop', 'waiting'],
|
|
['destroy', 'running']
|
|
])(
|
|
'suppresses fleet probes during %s (%s), preserves evidence, and resumes afterwards',
|
|
async (action, state) => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
|
const { rconPlutoniumVersion } =
|
|
await import('../../src/lib/server/integrations/rcon.js');
|
|
const saved = db()
|
|
.prepare('SELECT * FROM plutonium_versions WHERE server = ?')
|
|
.get(servers[0]);
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('busy', ?, ?, ?, 'Provisioning', ?, 'test')`
|
|
)
|
|
.run(servers[0], action, state, new Date().toISOString());
|
|
closeDb();
|
|
vi.mocked(nativeHealth).mockClear();
|
|
vi.mocked(rconPlutoniumVersion).mockClear();
|
|
vi.setSystemTime(Date.now() + 1_000);
|
|
await m.checkPlutonium();
|
|
expect(nativeHealth).toHaveBeenCalledTimes(1);
|
|
expect(nativeHealth).toHaveBeenCalledWith(servers[1]);
|
|
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1);
|
|
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[1]);
|
|
expect(
|
|
db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])
|
|
).toEqual(saved);
|
|
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'busy'").run();
|
|
await m.checkPlutonium();
|
|
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]);
|
|
expect(
|
|
db()
|
|
.prepare('SELECT checked_at FROM plutonium_versions WHERE server = ?')
|
|
.get(servers[0])
|
|
).toEqual({ checked_at: new Date().toISOString() });
|
|
}
|
|
);
|
|
it('suppresses fleet probes while a failed restart retains recovery capacity', async () => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const saved = db()
|
|
.prepare('SELECT * FROM plutonium_versions WHERE server = ?')
|
|
.get(servers[0]);
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('failed-restart', ?, 'restart', 'failed', 'Timed out', ?, 'test')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
db()
|
|
.prepare(
|
|
'INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)'
|
|
)
|
|
.run(servers[0], new Date().toISOString(), 'failed-restart');
|
|
vi.mocked(nativeHealth).mockClear();
|
|
vi.mocked(rconPlutoniumVersion).mockClear();
|
|
await m.checkPlutonium();
|
|
expect(nativeHealth).not.toHaveBeenCalledWith(servers[0]);
|
|
expect(rconPlutoniumVersion).not.toHaveBeenCalledWith(servers[0]);
|
|
expect(
|
|
db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])
|
|
).toEqual(saved);
|
|
db()
|
|
.prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?')
|
|
.run(new Date().toISOString(), servers[0]);
|
|
vi.setSystemTime(Date.now() + 1_000);
|
|
await m.checkPlutonium();
|
|
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]);
|
|
});
|
|
it.each(['health', 'version'] as const)(
|
|
'discards maintenance observations when provisioning is queued during the %s probe',
|
|
async (phase) => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
|
const { rconPlutoniumVersion } =
|
|
await import('../../src/lib/server/integrations/rcon.js');
|
|
const saved = db()
|
|
.prepare('SELECT * FROM plutonium_versions WHERE server = ?')
|
|
.get(servers[0]);
|
|
const queue = () =>
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('new-rebuild', ?, 'rebuild', 'waiting', 'Waiting', ?, 'test')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
vi.mocked(rconPlutoniumVersion).mockClear();
|
|
if (phase === 'health')
|
|
vi.mocked(nativeHealth).mockImplementationOnce(async () => {
|
|
queue();
|
|
return { ok: true, state: 'running' };
|
|
});
|
|
else
|
|
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
|
queue();
|
|
return { revision: null, error: 'Restarting' };
|
|
});
|
|
vi.setSystemTime(Date.now() + 1_000);
|
|
expect(await probeVersion(servers[0], { maintenance: true })).toBeNull();
|
|
if (phase === 'health') expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
|
expect(
|
|
db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])
|
|
).toEqual(saved);
|
|
}
|
|
);
|
|
it('coalesces published-release checks without probing the fleet', async () => {
|
|
const { checkRelease, release } =
|
|
await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const first = checkRelease();
|
|
const second = checkRelease();
|
|
expect(second).toBe(first);
|
|
await Promise.all([first, second]);
|
|
expect(fetch).toHaveBeenCalledTimes(1);
|
|
expect(release()).toMatchObject({ revision: 5354, error: null });
|
|
expect(nativeHealth).not.toHaveBeenCalled();
|
|
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
|
await checkRelease();
|
|
expect(fetch).toHaveBeenCalledTimes(2);
|
|
});
|
|
it('lets the verifying rebuild probe its own recovery while maintenance still yields', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion, runningVersions } =
|
|
await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const submittedAt = new Date(Date.now() - 1000).toISOString();
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
|
VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`
|
|
)
|
|
.run(servers[0], submittedAt, submittedAt);
|
|
db()
|
|
.prepare(
|
|
'INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)'
|
|
)
|
|
.run(servers[0], submittedAt, 'rebuild');
|
|
expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBe(5354);
|
|
expect(
|
|
db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])
|
|
).toEqual({ healthy_at: null });
|
|
// A successful revision check alone cannot advertise completed recovery.
|
|
expect(
|
|
runningVersions().find((version) => version.server === servers[0])?.revision
|
|
).toBeNull();
|
|
const saved = db()
|
|
.prepare('SELECT * FROM plutonium_versions WHERE server=?')
|
|
.get(servers[0]);
|
|
vi.mocked(rconPlutoniumVersion).mockClear();
|
|
expect(await probeVersion(servers[0], { maintenance: true })).toBeNull();
|
|
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
|
expect(
|
|
db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])
|
|
).toEqual(saved);
|
|
});
|
|
it.each(['another-owner', 'another-server', 'unsubmitted', 'finished'] as const)(
|
|
'does not bypass recovery for %s',
|
|
async (scenario) => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { rconPlutoniumVersion } =
|
|
await import('../../src/lib/server/integrations/rcon.js');
|
|
const submittedAt = new Date(Date.now() - 1000).toISOString();
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
|
VALUES ('rebuild', ?, 'rebuild', ?, 'Verifying', ?, ?, 'test')`
|
|
)
|
|
.run(
|
|
scenario === 'another-server' ? servers[1] : servers[0],
|
|
scenario === 'finished' ? 'done' : 'awaiting-health',
|
|
submittedAt,
|
|
scenario === 'unsubmitted' ? null : submittedAt
|
|
);
|
|
db()
|
|
.prepare(
|
|
'INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)'
|
|
)
|
|
.run(
|
|
servers[0],
|
|
submittedAt,
|
|
scenario === 'another-owner' ? 'other-rebuild' : 'rebuild'
|
|
);
|
|
expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull();
|
|
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
|
expect(
|
|
db()
|
|
.prepare('SELECT healthy_at FROM restart_rollout WHERE server=?')
|
|
.get(servers[0])
|
|
).toEqual({ healthy_at: null });
|
|
}
|
|
);
|
|
it('discards version evidence if its verifying operation finishes during the probe', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const submittedAt = new Date(Date.now() - 1000).toISOString();
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
|
VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`
|
|
)
|
|
.run(servers[0], submittedAt, submittedAt);
|
|
db()
|
|
.prepare(
|
|
'INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)'
|
|
)
|
|
.run(servers[0], submittedAt, 'rebuild');
|
|
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
|
db().prepare("UPDATE server_deployments SET state='failed' WHERE id='rebuild'").run();
|
|
return { revision: 5354, error: null };
|
|
});
|
|
expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull();
|
|
expect(
|
|
db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])
|
|
).toBeUndefined();
|
|
});
|
|
it('refreshes hourly fleet versions even when provisioning recently refreshed the release manifest', async () => {
|
|
const m = await api();
|
|
const { checkRelease, HOUR } =
|
|
await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
await m.checkPlutonium();
|
|
vi.setSystemTime(Date.now() + HOUR);
|
|
await checkRelease();
|
|
vi.mocked(rconPlutoniumVersion).mockClear();
|
|
await m.maintenanceTick();
|
|
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(servers.length);
|
|
for (const server of servers) expect(rconPlutoniumVersion).toHaveBeenCalledWith(server);
|
|
await m.maintenanceTick();
|
|
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(servers.length);
|
|
});
|
|
it('uses the official production manifest without requesting the challenged changelog page', async () => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
expect(fetch).toHaveBeenCalledTimes(1);
|
|
expect(fetch).toHaveBeenCalledWith(PLUTONIUM_MANIFEST_URL, expect.any(Object));
|
|
expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: null });
|
|
expect(parsePlutoniumManifest({ product: 'plutonium-core-prod', revision: 5354 })).toBe(
|
|
5354
|
|
);
|
|
for (const value of [
|
|
null,
|
|
{},
|
|
{ ref: '1.0.221' },
|
|
{ product: 'plutonium-core-dev', revision: 6000 },
|
|
{ product: 'plutonium-core-prod', revision: '5354' },
|
|
{ product: 'plutonium-core-prod', revision: -1 }
|
|
]) {
|
|
expect(() => parsePlutoniumManifest(value)).toThrow();
|
|
}
|
|
});
|
|
it('parses explicit running revisions without interpreting engine numbers as releases', () => {
|
|
expect(parsePlutoniumRevision('print\nPlutonium T6 r5354 win-x86')).toBe(5354);
|
|
expect(parsePlutoniumRevision('version 1.0 build 20260921')).toBeNull();
|
|
});
|
|
it('preserves the cached release with a visible error and clears failed running-version observations', async () => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
vi.mocked(fetch).mockRejectedValue(new Error('Offline'));
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Timeout' });
|
|
await m.checkPlutonium();
|
|
expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: 'Offline' });
|
|
expect(
|
|
m.plutoniumStatus().servers.every((s) => s.revision === null && s.error === 'Timeout')
|
|
).toBe(true);
|
|
});
|
|
it.each([
|
|
() => new Response('Blocked', { status: 403 }),
|
|
() => new Response('<html>Browser challenge</html>'),
|
|
() => Response.json({ product: 'plutonium-core-dev', revision: 9999 })
|
|
])(
|
|
'keeps the cached revision when the manifest is unavailable or invalid',
|
|
async (response) => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
vi.mocked(fetch).mockImplementation(async () => response());
|
|
await m.checkPlutonium();
|
|
expect(m.plutoniumStatus().release.revision).toBe(5354);
|
|
expect(m.plutoniumStatus().release.error).toBeTruthy();
|
|
m.queueMaintenance([servers[0]]);
|
|
expect(m.plutoniumStatus().jobs[0].target_revision).toBeNull();
|
|
}
|
|
);
|
|
it('rejects a version observed at the startup submission timestamp, then accepts a later observation', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion, runningVersions, checkRelease } =
|
|
await import('../../src/lib/server/integrations/plutonium.js');
|
|
const submittedAt = new Date().toISOString();
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
|
VALUES ('startup', ?, 'create', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`
|
|
)
|
|
.run(servers[0], submittedAt, submittedAt);
|
|
await checkRelease();
|
|
expect(await probeVersion(servers[0])).toBeNull();
|
|
expect(runningVersions()[0]).toMatchObject({
|
|
revision: null,
|
|
checked_at: submittedAt,
|
|
update_status: 'unknown',
|
|
error: expect.stringContaining('after startup')
|
|
});
|
|
vi.setSystemTime(Date.now() + 1);
|
|
expect(await probeVersion(servers[0])).toBe(5354);
|
|
expect(runningVersions()[0]).toMatchObject({
|
|
revision: 5354,
|
|
update_status: 'current',
|
|
error: null
|
|
});
|
|
});
|
|
it('discards a running revision when a new lifecycle submission arrives during its probe', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion, runningVersions } =
|
|
await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
|
const now = new Date().toISOString();
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
|
VALUES ('restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`
|
|
)
|
|
.run(servers[0], now, now);
|
|
return { revision: 5354, error: null };
|
|
});
|
|
expect(await probeVersion(servers[0])).toBeNull();
|
|
expect(runningVersions()[0]).toMatchObject({ revision: null, update_status: 'unknown' });
|
|
expect(
|
|
db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])
|
|
).toBeUndefined();
|
|
});
|
|
it.each(['startup', 'replacement'] as const)(
|
|
'preserves a newer successful revision when an old probe finishes after %s',
|
|
async (change) => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion, runningVersions, checkRelease } =
|
|
await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { rconPlutoniumVersion } =
|
|
await import('../../src/lib/server/integrations/rcon.js');
|
|
await checkRelease();
|
|
db()
|
|
.prepare('UPDATE servers SET created_at = ? WHERE name = ?')
|
|
.run('2026-09-21T05:58:00.000Z', servers[0]);
|
|
let finishOld!: (value: { revision: number; error: null }) => void;
|
|
let started!: () => void;
|
|
const probing = new Promise<void>((resolve) => {
|
|
started = resolve;
|
|
});
|
|
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(() => {
|
|
started();
|
|
return new Promise((resolve) => {
|
|
finishOld = resolve;
|
|
});
|
|
});
|
|
const oldProbe = probeVersion(servers[0]);
|
|
await probing;
|
|
const now = new Date().toISOString();
|
|
if (change === 'startup') {
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
|
VALUES ('new-startup', ?, 'start', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`
|
|
)
|
|
.run(servers[0], now, now);
|
|
} else {
|
|
// Both registrations have no lifecycle submission, so only their
|
|
// registry identity distinguishes the old and replacement instances.
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
|
|
db()
|
|
.prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)')
|
|
.run(servers[0], now);
|
|
}
|
|
vi.setSystemTime(Date.now() + 1);
|
|
expect(await probeVersion(servers[0])).toBe(5354);
|
|
const saved = db()
|
|
.prepare('SELECT * FROM plutonium_versions WHERE server = ?')
|
|
.get(servers[0]);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
finishOld({ revision: 5300, error: null });
|
|
expect(await oldProbe).toBeNull();
|
|
expect(
|
|
db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])
|
|
).toEqual(saved);
|
|
expect(runningVersions()[0]).toMatchObject({
|
|
revision: 5354,
|
|
error: null,
|
|
update_status: 'current'
|
|
});
|
|
}
|
|
);
|
|
it('discards a running revision when the server is stopped during its probe', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion, runningVersions } =
|
|
await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
|
db()
|
|
.prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?")
|
|
.run(servers[0]);
|
|
return { revision: 5354, error: null };
|
|
});
|
|
expect(await probeVersion(servers[0])).toBeNull();
|
|
expect(runningVersions()[0]).toMatchObject({
|
|
revision: null,
|
|
error: 'Server is intentionally stopped.',
|
|
update_status: 'stopped'
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('maintenance timezones', () => {
|
|
it.each([
|
|
['06:00', '2026-09-21T05:59:00Z', 'America/New_York', '2026-09-21T10:00:00.000Z'],
|
|
['06:00', '2026-12-21T05:59:00Z', 'America/New_York', '2026-12-21T11:00:00.000Z'],
|
|
['06:00', '2026-09-21T00:00:00Z', 'Asia/Kathmandu', '2026-09-21T00:15:00.000Z'],
|
|
['06:00', '2026-09-21T00:15:01Z', 'Asia/Kathmandu', '2026-09-22T00:15:00.000Z'],
|
|
['02:30', '2026-03-08T05:00:00Z', 'America/New_York', '2026-03-08T07:00:00.000Z'],
|
|
['01:30', '2026-11-01T04:00:00Z', 'America/New_York', '2026-11-01T05:30:00.000Z'],
|
|
['01:30', '2026-11-01T05:31:00Z', 'America/New_York', '2026-11-02T06:30:00.000Z'],
|
|
['02:15', '2026-10-03T13:00:00Z', 'Australia/Lord_Howe', '2026-10-03T15:30:00.000Z']
|
|
])('schedules %s after %s in %s', (time, now, zone, expected) => {
|
|
expect(nextMaintenanceAt(time, new Date(now), zone)).toBe(expected);
|
|
});
|
|
it('preserves legacy UTC schedules and displays their next occurrence in the user timezone', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare('INSERT INTO settings (key, value) VALUES (?, ?)')
|
|
.run(
|
|
'plutonium_schedule',
|
|
JSON.stringify({
|
|
enabled: true,
|
|
time: '06:00',
|
|
nextAt: '2026-09-21T06:00:00.000Z'
|
|
})
|
|
);
|
|
const m = await api();
|
|
const schedule = m.plutoniumStatus().schedule;
|
|
expect(schedule.timeZone).toBe('UTC');
|
|
expect(maintenanceTimeInZone(schedule, 'America/New_York')).toBe('02:00');
|
|
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-21T06:00:00.000Z');
|
|
});
|
|
});
|
|
|
|
describe('persistent maintenance', () => {
|
|
it('waits to attach legacy maintenance while another operation owns the server', async () => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(`INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)`)
|
|
.run(servers[0], new Date().toISOString());
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0]).toMatchObject({
|
|
state: 'queued',
|
|
provisioning_id: null,
|
|
detail: expect.stringContaining('another provisioning operation')
|
|
});
|
|
expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({
|
|
count: 1
|
|
});
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'")
|
|
.run();
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0].provisioning_id).toBeTruthy();
|
|
expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({
|
|
count: 2
|
|
});
|
|
});
|
|
it('defers completed maintenance verification while a later operation owns the server, then resumes', async () => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
m.queueMaintenance([servers[0]]);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const job = m.plutoniumStatus().jobs[0];
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?")
|
|
.run(job.provisioning_id);
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('later-restart', ?, 'restart', 'waiting', 'Waiting to restart', ?, 'test')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(nativeHealth).mockClear();
|
|
vi.mocked(rconPlutoniumVersion).mockClear();
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0]).toMatchObject({
|
|
state: 'queued',
|
|
detail: expect.stringContaining('before verifying')
|
|
});
|
|
expect(nativeHealth).not.toHaveBeenCalled();
|
|
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-restart'")
|
|
.run();
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
|
|
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]);
|
|
});
|
|
it('does not fail maintenance verification when another operation starts during its version probe', async () => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
m.queueMaintenance([servers[0]]);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?")
|
|
.run(m.plutoniumStatus().jobs[0].provisioning_id);
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('later-rebuild', ?, 'rebuild', 'running', 'Rebuilding', ?, 'test')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
return { revision: null, error: 'Restarting' };
|
|
});
|
|
const before = db()
|
|
.prepare('SELECT * FROM plutonium_versions WHERE server = ?')
|
|
.get(servers[0]);
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0]).toMatchObject({
|
|
state: 'queued',
|
|
detail: expect.stringContaining('before verifying')
|
|
});
|
|
expect(
|
|
db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])
|
|
).toEqual(before);
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-rebuild'")
|
|
.run();
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
|
|
});
|
|
it('mirrors its own failed operation even while another operation owns the server', async () => {
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
m.queueMaintenance([servers[0]]);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET state = 'failed', error = 'Restart failed' WHERE id = ?"
|
|
)
|
|
.run(m.plutoniumStatus().jobs[0].provisioning_id);
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('later-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0]).toMatchObject({
|
|
state: 'failed',
|
|
detail: 'Restart failed'
|
|
});
|
|
});
|
|
it('validates API requests without changing the schedule or queue', async () => {
|
|
const { POST } = await import('../../src/routes/api/plutonium/+server.js');
|
|
const post = (body: unknown) =>
|
|
POST({
|
|
request: new Request('http://localhost/api/plutonium', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(body)
|
|
})
|
|
});
|
|
expect((await post({ action: 'schedule', enabled: true, time: '26:99' })).status).toBe(400);
|
|
expect((await post({ action: 'restart', servers: ['missing'] })).status).toBe(400);
|
|
expect((await post({ action: 'cancel', id: 100 })).status).toBe(409);
|
|
expect((await post({ action: 'schedule', enabled: true, time: '06:00' })).status).toBe(400);
|
|
expect(
|
|
(await post({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'invalid' }))
|
|
.status
|
|
).toBe(400);
|
|
expect(
|
|
(
|
|
await post({
|
|
action: 'schedule',
|
|
enabled: true,
|
|
time: '06:00',
|
|
timeZone: 'America/New_York'
|
|
})
|
|
).status
|
|
).toBe(200);
|
|
const m = await api();
|
|
expect(m.plutoniumStatus().jobs).toEqual([]);
|
|
expect(m.plutoniumStatus().schedule.enabled).toBe(true);
|
|
expect(m.plutoniumStatus().schedule).toMatchObject({
|
|
timeZone: 'America/New_York',
|
|
nextAt: '2026-09-21T10:00:00.000Z'
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('Maintenance policies and runtime updates', () => {
|
|
async function outdated() {
|
|
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
|
|
const m = await api();
|
|
await m.checkPlutonium();
|
|
return m;
|
|
}
|
|
it('preserves the legacy recovery preference without enabling updates', async () => {
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("INSERT INTO settings(key,value) VALUES ('watchdog_enabled','0')").run();
|
|
const p = await import('../../src/lib/server/maintenance/policies.js');
|
|
expect(p.maintenancePolicies()).toEqual({
|
|
paused: false,
|
|
health: false,
|
|
drift: false,
|
|
updates: false
|
|
});
|
|
p.setMaintenancePolicy('health', true);
|
|
p.pauseMaintenance(true);
|
|
closeDb();
|
|
expect(p.maintenancePolicies()).toEqual({
|
|
paused: true,
|
|
health: true,
|
|
drift: false,
|
|
updates: false
|
|
});
|
|
expect(p.maintenanceAllowed('health')).toBe(false);
|
|
});
|
|
it('only automatically updates verified outdated servers and requires opt-in', async () => {
|
|
const m = await outdated();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare('UPDATE plutonium_versions SET revision = NULL WHERE server = ?')
|
|
.run(servers[1]);
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs).toEqual([]);
|
|
const p = await import('../../src/lib/server/maintenance/policies.js');
|
|
p.setMaintenancePolicy('updates', true);
|
|
await m.maintenanceTick();
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs).toEqual([
|
|
expect.objectContaining({
|
|
server: servers[0],
|
|
kind: 'update',
|
|
origin: 'automatic',
|
|
target_revision: 5354
|
|
})
|
|
]);
|
|
expect(
|
|
m.plutoniumStatus().servers.find((s) => s.server === servers[1])?.update_status
|
|
).toBe('unknown');
|
|
});
|
|
it.each(['queued', 'waiting', 'running', 'awaiting-health'])(
|
|
'skips automatic updates during %s provisioning and resumes after completion',
|
|
async (state) => {
|
|
const m = await outdated();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('existing-restart', ?, 'restart', ?, 'Restarting', ?, 'test')`
|
|
)
|
|
.run(servers[0], state, new Date().toISOString());
|
|
const p = await import('../../src/lib/server/maintenance/policies.js');
|
|
p.setMaintenancePolicy('updates', true);
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs).toEqual([
|
|
expect.objectContaining({ server: servers[1], kind: 'update', origin: 'automatic' })
|
|
]);
|
|
expect(
|
|
db()
|
|
.prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server = ?')
|
|
.get(servers[0])
|
|
).toEqual({ count: 1 });
|
|
db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET state = 'done' WHERE id = 'existing-restart'"
|
|
)
|
|
.run();
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs).toHaveLength(2);
|
|
expect(m.plutoniumStatus().jobs[0]).toMatchObject({
|
|
server: servers[0],
|
|
kind: 'update',
|
|
origin: 'automatic'
|
|
});
|
|
}
|
|
);
|
|
it('skips a busy server for the scheduled occurrence while other servers proceed', async () => {
|
|
const m = await outdated();
|
|
m.saveMaintenanceSchedule(true, '06:00', 'UTC');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
vi.setSystemTime(new Date('2026-09-21T06:00:00Z'));
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs).toEqual([
|
|
expect.objectContaining({ server: servers[1], kind: 'schedule' })
|
|
]);
|
|
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-22T06:00:00.000Z');
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'")
|
|
.run();
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs).toHaveLength(1);
|
|
vi.setSystemTime(new Date('2026-09-22T06:00:00Z'));
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs).toHaveLength(2);
|
|
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'schedule' });
|
|
});
|
|
it('skips automatic and scheduled requests while a failed operation retains recovery capacity', async () => {
|
|
const m = await outdated();
|
|
m.saveMaintenanceSchedule(true, '06:00', 'UTC');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('failed-restart', ?, 'restart', 'failed', 'Restart failed', ?, 'test')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, 'failed-restart')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(0);
|
|
vi.setSystemTime(new Date('2026-09-21T06:00:00Z'));
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs).toEqual([
|
|
expect.objectContaining({ server: servers[1], kind: 'schedule' })
|
|
]);
|
|
db()
|
|
.prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?')
|
|
.run(new Date().toISOString(), servers[0]);
|
|
expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(1);
|
|
});
|
|
it('preserves explicit update requests while another operation owns the server', async () => {
|
|
const m = await outdated();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`
|
|
)
|
|
.run(servers[0], new Date().toISOString());
|
|
expect(m.queueUpdates([servers[0]])).toBe(1);
|
|
expect(m.plutoniumStatus().jobs[0]).toMatchObject({
|
|
server: servers[0],
|
|
kind: 'update',
|
|
origin: 'manual'
|
|
});
|
|
});
|
|
it('does not treat stale, pre-restart, or unhealthy observations as current runtime evidence', async () => {
|
|
const m = await outdated();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare('INSERT INTO restart_rollout(server,requested_at,healthy_at) VALUES (?,?,?)')
|
|
.run(servers[0], new Date().toISOString(), new Date().toISOString());
|
|
expect(m.plutoniumStatus().servers[0].update_status).toBe('unknown');
|
|
vi.setSystemTime(Date.now() + 3_600_000);
|
|
expect(() => m.queueUpdates()).toThrow('published revision');
|
|
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconPlutoniumVersion).mockClear();
|
|
vi.mocked(nativeHealth).mockResolvedValue({
|
|
ok: false,
|
|
state: 'starting',
|
|
reason: 'Starting'
|
|
});
|
|
await m.checkPlutonium();
|
|
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
|
expect(m.plutoniumStatus().servers.every((s) => s.update_status === 'unknown')).toBe(true);
|
|
});
|
|
});
|