josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: unify recovery, updates, and schedules under Maintenance

Browse Source
This commit is contained in:
josh
2026-09-27 16:00:18 -04:00
parent 8ea696417f
commit fc46cb9da4
42 changed files with 1256 additions and 833 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/fleet-overview.test.ts
+10
View File
@@ -60,6 +60,16 @@ describe('persisted Fleet operations', () => {
beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); });
afterEach(async () => { await fixture.cleanup(); });
it('shows failed update verification even after a healthy restart and clears attention on a new attempt', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const handle = db();
handle.prepare("INSERT INTO maintenance_jobs(server,kind,state,target_revision,requested_at,detail) VALUES ('server','update','failed',5354,?,'Revision not verified')").run(iso());
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
expect(fleetOperations(now)).toEqual([expect.objectContaining({ kind: 'maintenance', state: 'attention', detail: 'Revision not verified' })]);
handle.prepare("INSERT INTO maintenance_jobs(server,kind,state,target_revision,requested_at) VALUES ('server','update','queued',5354,?)").run(iso(1000));
expect(fleetOperations(now)).toEqual([]);
});
it('recovers waiting and recovering campaigns without needing a live event', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const handle = db();
tests/unit/maintenance-api.test.ts
+39
View File
@@ -0,0 +1,39 @@
import { beforeEach, afterEach, describe, it, expect, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn().mockResolvedValue(undefined) }));
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); });
afterEach(async () => { await fixture.cleanup(); });
const request = (body: unknown) => new Request('http://localhost/api/maintenance', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
describe('Maintenance API', () => {
it('saves independent policies and the global pause without changing the restart schedule', async () => {
const { POST, GET } = await import('../../src/routes/api/maintenance/+server.js');
expect((await POST({ request: request({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'America/Indiana/Indianapolis' }) })).status).toBe(200);
const before = (await (GET()).json()).runtime.schedule;
for (const body of [{ action: 'policy', policy: 'health', enabled: false }, { action: 'policy', policy: 'updates', enabled: true }, { action: 'pause', paused: true }]) {
expect((await POST({ request: request(body) })).status).toBe(200);
}
const data = await GET().json();
expect(data.policies).toEqual({ paused: true, health: false, drift: true, updates: true });
expect(data.runtime.schedule).toEqual(before);
expect((await POST({ request: request({ action: 'update' }) })).status).toBe(400);
});
it('rejects malformed actions without changing policies or queuing work', async () => {
const { POST, GET } = await import('../../src/routes/api/maintenance/+server.js');
const before = await GET().json();
for (const body of [null, [], {}, { action: 'pause', paused: 'yes' }, { action: 'policy', policy: 'unknown', enabled: true },
{ action: 'policy', policy: 'updates', enabled: 1 }, { action: 'schedule', enabled: true, time: '25:00', timeZone: 'UTC' },
{ action: 'schedule', enabled: true, time: '06:00', timeZone: 'invalid' }, { action: 'update', servers: [] },
{ action: 'update', servers: [1] }, { action: 'retry', id: '1' }]) {
expect((await POST({ request: request(body) })).status).toBe(400);
}
expect(await GET().json()).toEqual(before);
});
it('keeps old Watchdog URLs working through query-preserving redirects', async () => {
const index = await import('../../src/routes/watchdog/+page.server.js');
const detail = await import('../../src/routes/watchdog/[id]/+page.server.js');
expect(() => index.load({ url: new URL('http://localhost/watchdog?kind=drift&server=test') })).toThrow(expect.objectContaining({ status: 308, location: '/maintenance?kind=drift&server=test' }));
expect(() => detail.load({ params: { id: 'abc' }, url: new URL('http://localhost/watchdog/abc?test=1') })).toThrow(expect.objectContaining({ status: 308, location: '/maintenance/abc?test=1' }));
});
});
tests/unit/plutonium.test.ts
+114
View File
@@ -233,3 +233,117 @@ describe('persistent maintenance', () => {
expect(m.plutoniumStatus().schedule.nextAt).toBeNull();
});
});
describe('Maintenance policies and runtime updates', () => {
async function outdated() {
const rcon = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
const m = await api(); await m.checkPlutonium(); return m;
}
it('preserves the legacy recovery preference without enabling updates', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().prepare("INSERT INTO settings(key,value) VALUES ('watchdog_enabled','0')").run();
const p = await import('../../src/lib/server/maintenance/policies.js');
expect(p.maintenancePolicies()).toEqual({ paused: false, health: false, drift: false, updates: false });
p.setMaintenancePolicy('health', true); p.pauseMaintenance(true); closeDb();
expect(p.maintenancePolicies()).toEqual({ paused: true, health: true, drift: false, updates: false });
expect(p.maintenanceAllowed('health')).toBe(false);
});
it('only automatically updates verified outdated servers and requires opt-in', async () => {
const m = await outdated();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE plutonium_versions SET revision = NULL WHERE server = ?').run(servers[1]);
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toEqual([]);
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
await m.maintenanceTick(); await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[0], kind: 'update', origin: 'automatic', target_revision: 5354 })]);
expect(m.plutoniumStatus().servers.find(s => s.server === servers[1])?.update_status).toBe('unknown');
});
it('holds queued automatic updates when their policy is disabled, but finishes submitted recovery while paused', async () => {
const m = await outdated();
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
m.queueUpdates([servers[0]], true); p.setMaintenancePolicy('updates', false);
await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
p.setMaintenancePolicy('updates', true); p.pauseMaintenance(true);
await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
p.pauseMaintenance(false); await processProvisioning(); expect(komodoRestart).toHaveBeenCalledTimes(1);
p.pauseMaintenance(true); vi.setSystemTime(Date.now() + 15_000);
await processProvisioning(); await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', operation_state: 'done' });
expect(komodoRestart).toHaveBeenCalledTimes(1);
});
it('latches failed update verification across restarts and retention, and supports an explicit retry', async () => {
const m = await outdated();
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[1]);
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
const failed = m.plutoniumStatus().jobs[0];
expect(failed).toMatchObject({ kind: 'update', state: 'failed', operation_state: 'done' });
closeDb(); vi.setSystemTime(Date.now() + 31 * 86_400_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(1);
expect(m.retryMaintenance(failed.id)).toBe(1);
expect(m.plutoniumStatus().jobs).toHaveLength(2);
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', origin: 'manual' });
expect(db().prepare("SELECT 1 FROM server_deployments WHERE action = 'rebuild'").get()).toBeUndefined();
});
it('does not restart intentionally stopped servers, including jobs queued before the intent changed', async () => {
const m = await outdated(); m.queueUpdates([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE servers SET desired_state = 'stopped'").run();
await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
expect(m.queueUpdates()).toBe(0);
expect(m.plutoniumStatus().servers.every(s => s.update_status === 'stopped')).toBe(true);
});
it('separates scheduled restart success from revision verification and coalesces paused runs', async () => {
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC');
const p = await import('../../src/lib/server/maintenance/policies.js'); p.pauseMaintenance(true);
vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(0);
p.pauseMaintenance(false); await maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(2);
expect(m.plutoniumStatus().jobs.every(j => j.kind === 'schedule' && j.target_revision === null)).toBe(true);
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Unknown reply' });
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs.find(j => j.server === servers[0])?.state).toBe('done');
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
});
it('does not treat stale, pre-restart, or unhealthy observations as current runtime evidence', async () => {
const m = await outdated();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('INSERT INTO restart_rollout(server,requested_at,healthy_at) VALUES (?,?,?)').run(servers[0], new Date().toISOString(), new Date().toISOString());
expect(m.plutoniumStatus().servers[0].update_status).toBe('unknown');
vi.setSystemTime(Date.now() + 3_600_000);
expect(() => m.queueUpdates()).toThrow('published revision');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockClear();
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
await m.checkPlutonium();
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
expect(m.plutoniumStatus().servers.every(s => s.update_status === 'unknown')).toBe(true);
});
it('adopts existing maintenance jobs without changing their IDs, schedules, or submitted operations', async () => {
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'America/New_York');
m.queueMaintenance([servers[0]]); await maintenanceTick();
const before = m.plutoniumStatus();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().exec('ALTER TABLE maintenance_jobs DROP COLUMN kind; ALTER TABLE maintenance_jobs DROP COLUMN origin;'); closeDb();
const after = m.plutoniumStatus();
expect(after.schedule).toEqual(before.schedule);
expect(after.jobs[0]).toMatchObject({ id: before.jobs[0].id, provisioning_id: before.jobs[0].provisioning_id, kind: 'legacy', origin: 'manual' });
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).toHaveBeenCalledTimes(1);
});
});
tests/unit/watchdog.test.ts
+33
View File
@@ -395,3 +395,36 @@ describe('Watchdog drift campaigns', () => {
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
});
});
describe('Maintenance recovery policy boundaries', () => {
it('separates health recovery from drift repair and holds an already queued drift rebuild', async () => {
const policies = await import('../../src/lib/server/maintenance/policies.js');
policies.setMaintenancePolicy('health', false);
down();
for (let n = 0; n < 3; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
expect(operations()).toEqual([]);
expect(onlyIncident().kind).toBe('health');
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'drift');
await watchdog.watchdogTick(true);
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
policies.setMaintenancePolicy('drift', false);
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
expect(operations()[0].state).toBe('waiting');
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
});
it('global pause blocks escalation without disabling the saved health policy', async () => {
await sustainedFailure();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
const policies = await import('../../src/lib/server/maintenance/policies.js');
policies.pauseMaintenance(true);
advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true);
expect(operations()).toHaveLength(1);
expect(policies.maintenancePolicies().health).toBe(true);
policies.pauseMaintenance(false); await watchdog.watchdogTick(true);
expect(operations()).toHaveLength(2);
expect(operations()[1].action).toBe('rebuild');
});
});