Add confirmed admin override for provisioning player checks

tests/unit/auth-access.test.ts

@@ -37,7 +37,7 @@ describe('management route permissions', () => {
         expect(allows('admin', route, method)).toBe(true);
     });
 
-    it.each(['/api/users', '/api/users/[id]', '/api/users/[id]/password', '/settings', '/settings/users', '/server-keys', '/keys', '/api/server-keys'])('reserves all methods on %s for administrators', route => {
+    it.each(['/api/users', '/api/users/[id]', '/api/users/[id]/password', '/settings', '/settings/users', '/server-keys', '/keys', '/api/server-keys', '/api/provisioning/[id]/override'])('reserves all methods on %s for administrators', route => {
         for (const method of ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE']) {
             expect(routePermission(route, method)).toBe('admin');
             expect(allows('viewer', route, method)).toBe(false);

tests/unit/native-provisioning.test.ts

@@ -463,6 +463,23 @@ describe('native instance provisioning', () => {
         await advance();
         expect(nativeInstance(job.name)?.generation).toBe(2);
     });
+    it('carries confirmed admin override through serialized rebuild preparation', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        heartbeat(hostId, { ...inventory(), capabilities: ['instance.safety-override'] });
+        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
+        remote.players = 2; await advance();
+        expect(await operationFor(rebuild.deploymentId)).toBeNull();
+        const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
+        expect(enableSafetyOverride(rebuild.deploymentId, { id: 'admin', username: 'admin' })).toEqual({ ok: true });
+        await advance();
+        expect(await operationFor(rebuild.deploymentId)).toMatchObject({ kind: 'instance.rebuild', generation: 1,
+            payload: { nextGeneration: 2, libraryId, safetyOverride: true } });
+        await acknowledge(rebuild.deploymentId);
+        expect((await detail(rebuild.deploymentId)).state).toBe('done');
+    });
+
     it('holds a rebuild when players join while its launch files are being prepared', async () => {
         const job = await queue(); await advance(); await acknowledge(job.deploymentId);
         const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');

tests/unit/native-safety.test.ts

@@ -6,7 +6,7 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
     rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } })),
     rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
 }));
-vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
 
 let fixture: Fixture; let hostId: string; let instanceId: string;
 const server = 't5_zombie_theater_1';
@@ -17,7 +17,7 @@ beforeEach(async () => {
     const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js');
     hostId = createHost({ name: 'Safety test host' }).id;
     enrollHost(createEnrollment(hostId).token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
-        storageAvailable: 100_000_000_000, runtimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }] });
+        storageAvailable: 100_000_000_000, runtimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }], capabilities: ['instance.safety-override'] });
     const { db } = await import('../../src/lib/server/db/index.js');
     instanceId = randomUUID();
     db().prepare("INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'t5','zombie_theater','zombie','theater',28960,'running')").run(server);
@@ -120,6 +120,133 @@ describe('native disruption safety at dispatch', () => {
     });
 });
 
+describe('provisioning admin override', () => {
+    const actor = { id: 'admin-id', username: 'recovery-admin' };
+    async function confirm(id: string) {
+        const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
+        expect(enableSafetyOverride(id, actor)).toEqual({ ok: true });
+    }
+
+    it.each(['restart', 'stop', 'destroy'] as const)('dispatches %s with unavailable player evidence only after confirmation', async action => {
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
+        const result = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, action);
+        if (!result.ok || !result.deploymentId) throw new Error('Request failed');
+        const id = result.deploymentId;
+        const status = await import('../../src/lib/server/fleet/status-evidence.js');
+        vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ players: null, source: null, observedAt: new Date().toISOString() });
+        await advance();
+        expect(await operationId(id)).toBeNull();
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        expect(getDeployment(id)).toMatchObject({ canOverrideSafety: true, safetyOverride: null });
+        await confirm(id);
+        await confirm(id);
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        closeDb(); // Approval and attribution survive controller restart.
+        await advance();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        const commandId = (await operationId(id))!;
+        expect(getHostOperation(commandId)).toMatchObject({ kind: `instance.${action === 'destroy' ? 'remove' : action}`, payload: { safetyOverride: true } });
+        expect(getDeployment(id)).toMatchObject({ canOverrideSafety: false, safetyOverride: { actorId: actor.id, actorName: actor.username } });
+        expect(db().prepare("SELECT COUNT(*) n FROM server_deployment_events WHERE deployment_id=? AND kind='safety-override'").get(id)).toEqual({ n: 1 });
+        await advance();
+        expect(await operationId(id)).toBe(commandId);
+    });
+
+    it('bypasses occupied telemetry while retaining health verification after the restart', async () => {
+        const id = await waitingRestart();
+        const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
+        saveTelemetry({ event: 'game_start', server, map: 'zombie_theater', players: [{ name: 'Playing', guid: '1' }], receivedAt: new Date().toISOString() });
+        await advance();
+        expect(await operationId(id)).toBeNull();
+        await confirm(id);
+        await advance();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, (await operationId(id))!, { state: 'completed', result: {} });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockResolvedValue({ state: 'running', healthy: false } as never);
+        await advance();
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        expect(getDeployment(id)?.deployment.state).toBe('awaiting-health');
+        expect(getDeployment(id)?.progress?.checks).toContainEqual(expect.objectContaining({ key: 'container', result: 'waiting' }));
+    });
+
+    it('retains recovery capacity limits and cancellation after approval', async () => {
+        const id = await waitingRestart();
+        await confirm(id);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 });
+        db().prepare("INSERT INTO servers(name) VALUES('other')").run();
+        db().prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('other',?)").run(new Date().toISOString());
+        await advance();
+        expect(await operationId(id)).toBeNull();
+        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
+        expect(cancelProvisioningJob(id)).toBe(true);
+        db().prepare("DELETE FROM restart_rollout WHERE server='other'").run();
+        await advance();
+        expect(await operationId(id)).toBeNull();
+    });
+
+    it('rejects old agents, stale instances, unsupported actions and submitted or terminal operations', async () => {
+        const id = await waitingRestart();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { enableSafetyOverride, safetyOverrideAvailability } = await import('../../src/lib/server/provisioning/safety-override.js');
+        db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json('[]')) WHERE id=?").run(hostId);
+        expect(enableSafetyOverride(id, actor)).toMatchObject({ ok: false, error: expect.stringContaining('Update') });
+        for (const state of ['running', 'awaiting-health', 'failed', 'interrupted', 'cancelled', 'done']) {
+            db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, id);
+            expect(safetyOverrideAvailability(id).applicable).toBe(false);
+        }
+        db().prepare("UPDATE server_deployments SET state='waiting',action='start' WHERE id=?").run(id);
+        expect(safetyOverrideAvailability(id).applicable).toBe(false);
+        db().prepare("UPDATE server_deployments SET action='restart',submitted_at='2026-10-05' WHERE id=?").run(id);
+        expect(safetyOverrideAvailability(id).applicable).toBe(false);
+        db().prepare('UPDATE server_deployments SET submitted_at=NULL WHERE id=?').run(id);
+        db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId);
+        expect(safetyOverrideAvailability(id).applicable).toBe(false);
+        expect(enableSafetyOverride('missing', actor).ok).toBe(false);
+    });
+
+    it('rechecks approval after an in-flight inspection before dispatch', async () => {
+        const id = await waitingRestart();
+        const pending = await suspendInspection();
+        await confirm(id);
+        pending.release(); await pending.running;
+        expect(await operationId(id)).toBeNull();
+        await advance();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ safetyOverride: true });
+    });
+
+    it('overrides a waiting campaign restart while keeping its staged changes attached', async () => {
+        const id = await stageCampaign();
+        const status = await import('../../src/lib/server/fleet/status-evidence.js');
+        vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ players: null, source: null, observedAt: new Date().toISOString() });
+        await advance();
+        expect(await operationId(id)).toBeNull();
+        await confirm(id);
+        await advance();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ stageId: id, safetyOverride: true });
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_id='campaign-race'").get()).toEqual({ state: 'applying' });
+    });
+
+    it('requires an administrator and explicit confirmation at the API boundary', async () => {
+        const id = await waitingRestart();
+        const { POST } = await import('../../src/routes/api/provisioning/[id]/override/+server.js');
+        const request = (role: string | null, confirmed: unknown) => POST({ params: { id }, locals: { user: role ? { ...actor, role } : null },
+            request: new Request(`http://localhost/api/provisioning/${id}/override`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ confirmed }) })
+        } as Parameters<typeof POST>[0]);
+        for (const role of [null, 'viewer', 'operator']) expect((await request(role, true)).status).toBe(403);
+        for (const value of [undefined, false, 'true', 1]) expect((await request('admin', value)).status).toBe(400);
+        expect((await request('admin', true)).status).toBe(202);
+        const { safetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
+        expect(safetyOverride(id)?.actorId).toBe(actor.id);
+    });
+});
+
 describe('native scheduler ownership', () => {
     it('rejects lifecycle commands for a registry row without a host instance', async () => {
         const { db } = await import('../../src/lib/server/db/index.js');

tests/unit/provisioning-refresh.test.ts

@@ -23,6 +23,16 @@ function deferred() {
 }
 
 describe('provisioning detail refresh ownership', () => {
+    it('sends explicit override confirmation and refreshes the operation after acceptance', async () => {
+        const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true })).mockResolvedValueOnce(response('a', 2));
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'));
+        await page.act('override', 'Override enabled.', { confirmed: true });
+        expect(fetcher).toHaveBeenNthCalledWith(1, '/api/provisioning/a/override', expect.objectContaining({
+            method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{"confirmed":true}'
+        }));
+        expect(page.state()).toMatchObject({ detail: detail('a', 2), notice: 'Override enabled.', busy: null });
+    });
     it('uses the server snapshot without another initial request and coalesces background reads', async () => {
         const pending = deferred();
         const fetcher = vi.fn(() => pending.promise);