Rebuild missing containers during maintenance recovery

2026-10-08 13:33:01 -04:00
parent 368e457dc6
commit b3fa486886
9 changed files with 201 additions and 25 deletions
@@ -51,6 +51,7 @@ const onlyIncident = () => { expect(incidents()).toHaveLength(1); return inciden
 const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[];
 function advance(ms: number) { vi.setSystemTime(Date.now() + ms); refreshHost(); }
 function down() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
+function missing() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'missing', reason: 'Container missing' }); }
 async function pollStatus() {
    const worker = await vi.importActual<typeof import('../../src/lib/server/pipeline/worker.js')>('../../src/lib/server/pipeline/worker.js');
    await worker.fanoutStatus();
@@ -95,6 +96,98 @@ describe('Watchdog detection and ownership', () => {
        expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0);
    });

+    it.each([false, true])('rebuilds a confirmed missing container once after three samples over two minutes (manual=%s)', async manual => {
+        missing();
+        const drift = await import('../../src/lib/server/fleet/drift.js');
+        const scan = vi.spyOn(drift, 'prepareDriftCampaign');
+        for (let sample = 0; sample < 3; sample++) {
+            if (!manual) await pollStatus();
+            await watchdog.watchdogTick(manual);
+            expect(onlyIncident().failures).toBe(sample + 1);
+            if (sample < 2) {
+                expect(operations()).toEqual([]);
+                advance(watchdog.WATCHDOG_SCAN_MS);
+            }
+        }
+        const incident = onlyIncident();
+        const queued = operations();
+        expect(incident).toMatchObject({ kind: 'health', state: 'recovering', failures: 3 });
+        expect(queued).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
+        expect(watchdog.watchdogDetail(incident.id)?.actions).toEqual([
+            expect.objectContaining({ action: 'rebuild', operation_id: queued[0].id, campaign_id: null })
+        ]);
+        expect(database.db().prepare('SELECT kind,source_id FROM provisioning_sources WHERE operation_id=?').all(queued[0].id))
+            .toEqual([{ kind: 'watchdog', source_id: incident.id }]);
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        expect(scan).not.toHaveBeenCalled();
+        expect(backend.nativeRequest).not.toHaveBeenCalled();
+        database.closeDb();
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual(queued);
+        expect(watchdog.watchdogDetail(incident.id)?.actions).toHaveLength(1);
+    });
+
+    it('does not accelerate missing-container recovery through repeated manual checks', async () => {
+        missing();
+        await watchdog.watchdogTick(true);
+        for (let check = 0; check < 5; check++) await watchdog.watchdogTick(true);
+        expect(onlyIncident().failures).toBe(1);
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        advance(watchdog.WATCHDOG_SCAN_MS - 1); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([]);
+        advance(1); await watchdog.watchdogTick(true);
+        expect(onlyIncident().failures).toBe(3);
+        expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
+    });
+
+    it('dispatches missing-container recovery as a rebuild without requesting an empty-player count', async () => {
+        missing();
+        for (let sample = 0; sample < 3; sample++) {
+            await watchdog.watchdogTick(true);
+            if (sample < 2) advance(watchdog.WATCHDOG_SCAN_MS);
+        }
+        const operation = operations()[0];
+        expect(operation).toMatchObject({ action: 'rebuild', state: 'waiting' });
+        const readFiles = vi.mocked(backend.nativeRequest).getMockImplementation()!;
+        vi.mocked(backend.nativeRequest).mockImplementation(async (instance, kind, payload) => {
+            if (kind === 'instance.inspect') return { exists: false, healthy: false, state: 'missing' } as never;
+            return readFiles(instance, kind, payload);
+        });
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        const binding = database.db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(operation.id) as { operation_id: string | null };
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        expect(binding.operation_id).toEqual(expect.any(String));
+        expect(getHostOperation(binding.operation_id!)).toMatchObject({ kind: 'instance.rebuild', generation: 1,
+            payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
+        expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]);
+        expect(rcon.rconGetPlayerCount).not.toHaveBeenCalled();
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+    });
+
+    it.each(['health disabled', 'maintenance paused'])('keeps missing containers observational with %s even when files have drifted', async policy => {
+        missing();
+        const policies = await import('../../src/lib/server/maintenance/policies.js');
+        policies.setMaintenancePolicy('drift', true);
+        if (policy === 'health disabled') policies.setMaintenancePolicy('health', false);
+        else policies.pauseMaintenance(true);
+        await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
+        const drift = await import('../../src/lib/server/fleet/drift.js');
+        const scan = vi.spyOn(drift, 'prepareDriftCampaign');
+        for (let sample = 0; sample < 4; sample++) {
+            await watchdog.watchdogTick(true);
+            advance(watchdog.WATCHDOG_SCAN_MS);
+        }
+        expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 4 });
+        expect(operations()).toEqual([]);
+        expect(scan).not.toHaveBeenCalled();
+        expect(backend.nativeRequest).not.toHaveBeenCalled();
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        policies.setMaintenancePolicy('health', true); policies.pauseMaintenance(false);
+        await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
+    });
+
    it('does not count the same sample again or use stale data, and resets after a monitoring gap', async () => {
        down();
        await pollStatus();
@@ -257,8 +350,12 @@ describe('Watchdog detection and ownership', () => {
        expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]);
    });

-    it('does not restart an unknown host agent state even after repeated sustained observations', async () => {
-        vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, reason: 'host agent request timed out' });
+    it.each([
+        ['unavailable', undefined, 'host agent request timed out'],
+        ['unknown', 'unknown', 'Agent could not inspect the container'],
+        ['offline', 'unknown', 'Host agent is disconnected']
+    ] as const)('does not recover an %s host agent state even after repeated sustained observations', async (_label, state, reason) => {
+        vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state, reason });
        for (let n = 0; n < 8; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
        expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' });
        expect(operations()).toEqual([]);
@@ -323,13 +420,13 @@ describe('Watchdog detection and ownership', () => {
        expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
    });

-    it('preserves intentional stop intent after completion and database reopening', async () => {
+    it.each(['exited', 'missing'])('preserves intentional stop intent for %s containers after completion and database reopening', async state => {
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        const stop = requestLifecycle(server, 'stop');
        expect(stop.ok).toBe(true);
        if (!stop.ok) throw new Error(stop.error);
        database.db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ? WHERE id = ?").run(new Date().toISOString(), stop.deploymentId);
-        down();
+        if (state === 'missing') missing(); else down();
        database.closeDb();
        for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
        expect(incidents()).toEqual([]);
@@ -354,11 +451,11 @@ describe('Watchdog detection and ownership', () => {
        expect(backend.nativeHealth).not.toHaveBeenCalled();
    });

-    it('rechecks lifecycle ownership after a health probe yields to an operator stop', async () => {
+    it.each(['exited', 'missing'])('rechecks lifecycle ownership after a %s probe yields to an operator stop', async state => {
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
            expect(requestLifecycle(server, 'stop').ok).toBe(true);
-            return { ok: false, state: 'exited', reason: 'Container exited' };
+            return { ok: false, state, reason: `Container ${state}` };
        });
        await watchdog.watchdogTick(true);
        expect(incidents()).toEqual([]);
@@ -412,6 +509,31 @@ describe('Watchdog detection and ownership', () => {

 describe('Watchdog recovery persistence', () => {

+    it('escalates a submitted restart that becomes missing after its five-minute recovery window', async () => {
+        const incident = await sustainedFailure();
+        const restart = operations()[0];
+        const submittedAt = new Date().toISOString();
+        database.db().prepare("UPDATE server_deployments SET state='awaiting-health',submitted_at=? WHERE id=?")
+            .run(submittedAt, restart.id);
+        database.db().prepare('INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES(?,?,?)')
+            .run(server, restart.id, submittedAt);
+        missing();
+        vi.mocked(rcon.rconStatusResult).mockClear();
+        advance(5 * 60_000 - 1); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([{ ...restart, state: 'awaiting-health' }]);
+        advance(1); await watchdog.watchdogTick(true);
+        const rebuilt = operations();
+        expect(rebuilt).toEqual([
+            { ...restart, state: 'failed' }, expect.objectContaining({ action: 'rebuild', state: 'waiting' })
+        ]);
+        expect(watchdog.watchdogDetail(incident.id)?.actions.map(action => action.action)).toEqual(['restart', 'rebuild']);
+        expect(database.db().prepare('SELECT operation_id FROM restart_rollout WHERE server=?').get(server))
+            .toEqual({ operation_id: rebuilt[1].id });
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual(rebuilt);
+    });
+
    it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
        down();
        await watchdog.watchdogTick(true);