Add confirmed admin override for provisioning player checks

src/lib/server/auth/access.ts

@@ -29,6 +29,7 @@ export function routePermission(routeId: string | null, method: string): RoutePe
         || within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin';
 
     if (!reading && ['/api/hosts', '/api/settings', '/api/game-files'].some(base => within(routeId, base))) return 'admin';
+    if (routeId === '/api/provisioning/[id]/override') return 'admin';
     // These endpoints mix operational actions with global policy and schedule
     // changes. Per-incident/job actions remain available to operators.
     if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) return 'admin';

src/lib/server/backends/schema.ts

@@ -23,6 +23,10 @@ export function migrateNativeInstances(handle: Database.Database): void {
             applied_at TEXT, verified_at TEXT
         );
         CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id);
+        CREATE TABLE IF NOT EXISTS provisioning_safety_overrides (
+            deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
+            actor_id TEXT NOT NULL, actor_name TEXT NOT NULL, confirmed_at TEXT NOT NULL
+        );
         CREATE TABLE IF NOT EXISTS ec_host_migrations (
             deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
             source_host_id TEXT NOT NULL, instance_id TEXT NOT NULL REFERENCES native_instances(id),

src/lib/server/provisioning/history.ts

@@ -11,6 +11,7 @@ import { workerActivity } from '../pipeline/activity.js';
 import { canCancelCreation } from './cancel-creation.js';
 import { canCancelProvisioningJob, provisioningCancelled } from './cancellation.js';
 import { nativeDeployment } from '../backends/native.js';
+import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
 import { serverTitle } from '../../ui/server-labels.js';
 import { provisioningPageSize, provisioningPageOffset, type ProvisioningListOptions, type ProvisioningListPage, type ProvisioningListRow } from '../../provisioning-list.js';
 
@@ -121,7 +122,9 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             SELECT collection_id, deployment_id FROM server_collection_members
             UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
             JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
+        const override = safetyOverrideAvailability(id);
         return { deployment, backend: 'native' as const, agentOperation: agentOperation ?? null,
+            canOverrideSafety: override.applicable, safetyOverrideUnavailableReason: override.unavailableReason, safetyOverride: safetyOverride(id),
             canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
             workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,

src/lib/server/provisioning/launch-preparation.ts

@@ -14,6 +14,7 @@ export interface NativeLaunchInput {
     serverKey: string;
     rconPassword: string;
     rebuild: boolean;
+    safetyOverride?: boolean;
 }
 
 export interface PreparedNativeLaunch {

src/lib/server/provisioning/launch-worker.ts

@@ -30,7 +30,7 @@ parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeL
                 PLUTO_SERVER_KEY: input.serverKey },
             files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path),
                 contentBase64: file.content, sha256: file.sha256 })),
-            ...(input.rebuild ? { nextGeneration: instance.generation + 1 } : {})
+            ...(input.rebuild ? { nextGeneration: instance.generation + 1, ...(input.safetyOverride ? { safetyOverride: true } : {}) } : {})
         };
         const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value));
         const recipeJson = encode(recipe);

src/lib/server/provisioning/native.ts

@@ -30,6 +30,7 @@ import type { ProvisioningAction } from '../../provisioning.js';
 import { hostAgentUpdatesEnabled } from '../../hosts.js';
 import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
 import { hostMigration, processMigrationRemoval, reconcileHostDrains } from './host-drain.js';
+import { safetyOverride, safetyOverrideCapability } from './safety-override.js';
 
 export type NativeSelection = import('../../hosts.js').NativePlacement;
 type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
@@ -565,6 +566,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     const reason = eligibility(job);
     if (reason) { hold(job, reason); return; }
+    const overrideSafety = !!safetyOverride(job.id);
+    if (overrideSafety && !host.inventory.capabilities?.includes(safetyOverrideCapability)) {
+        hold(job, 'Update this host’s agent before using admin override.'); return;
+    }
     if (job.action === 'destroy' || job.action === 'rebuild') {
         const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; }
     }
@@ -596,6 +601,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
     const checkOccupancy = async (): Promise<boolean> => {
         if (!stillActive(job)) return false;
+        if (overrideSafety) return true;
         // Fast controller check keeps occupied lobbies out of the operation slots.
         // The agent repeats this immediately before its local mutation.
         deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
@@ -630,7 +636,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
             prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
                 instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
-                rebuild: job.action === 'rebuild' && !migration });
+                rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety });
             if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
         }
         const commit = db().transaction(() => {
@@ -640,6 +646,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
             const liveHost = getHost(instance!.host_id);
             if (!liveHost?.connected || liveHost.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; }
+            // Confirmation can arrive while preflight or file preparation is awaiting I/O.
+            if (!!safetyOverride(job.id) !== overrideSafety) return;
+            if (overrideSafety && !liveHost.inventory.capabilities?.includes(safetyOverrideCapability)) {
+                hold(job, 'Update this host’s agent before using admin override.'); return;
+            }
             if (job.action === 'destroy' || job.action === 'rebuild') {
                 const beta = activeBetaRemovalBlocker(job.server);
                 if (beta) { hold(job, beta); return; }
@@ -665,8 +676,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
                     DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
             }
-            recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
+            recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: overrideSafety
+                ? 'Host eligibility checked; player safety checks bypassed by admin override' : 'Host eligibility and safety checks passed' } });
             const payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
+            if (overrideSafety) payload.safetyOverride = true;
             if (prepared) {
                 deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
                     checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });

src/lib/server/provisioning/safety-override.ts

@@ -0,0 +1,44 @@
+import { db } from '../db/index.js';
+import { getHost } from '../hosts/index.js';
+import { recordProgress } from './progress.js';
+import type { ProvisioningDetail } from '../../provisioning-detail.js';
+
+export const safetyOverrideCapability = 'instance.safety-override';
+
+export function safetyOverride(id: string): ProvisioningDetail['safetyOverride'] {
+    return db().prepare(`SELECT actor_id AS actorId, actor_name AS actorName, confirmed_at AS confirmedAt
+        FROM provisioning_safety_overrides WHERE deployment_id=?`).get(id) as ProvisioningDetail['safetyOverride'] ?? null;
+}
+
+/** Approval belongs to one unsubmitted operation and its original live instance. */
+export function safetyOverrideAvailability(id: string): { applicable: boolean; unavailableReason: string | null } {
+    const target = db().prepare(`SELECT i.host_id FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.generation=n.generation AND i.retired_at IS NULL
+        WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action IN ('restart','stop','rebuild','destroy')
+        AND d.state IN ('queued','waiting') AND d.submitted_at IS NULL AND n.operation_id IS NULL
+        AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id)
+        AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.cleanup_id=d.id)
+        AND NOT EXISTS (SELECT 1 FROM provisioning_safety_overrides o WHERE o.deployment_id=d.id)`)
+        .get(id) as { host_id: string } | undefined;
+    if (!target) return { applicable: false, unavailableReason: null };
+    return { applicable: true, unavailableReason: getHost(target.host_id)?.inventory.capabilities?.includes(safetyOverrideCapability)
+        ? null : 'Update this host’s agent before using admin override.' };
+}
+
+export function enableSafetyOverride(id: string, actor: { id: string; username: string }): { ok: true } | { ok: false; error: string } {
+    return db().transaction(() => {
+        // A retried confirmation never sends or repeats a command.
+        if (safetyOverride(id)) return { ok: true as const };
+        const availability = safetyOverrideAvailability(id);
+        if (!availability.applicable) return { ok: false as const, error: 'This operation can no longer be overridden. Refresh to see its current state.' };
+        if (availability.unavailableReason) return { ok: false as const, error: availability.unavailableReason };
+        const confirmedAt = new Date().toISOString();
+        db().prepare('INSERT INTO provisioning_safety_overrides(deployment_id,actor_id,actor_name,confirmed_at) VALUES(?,?,?,?)')
+            .run(id, actor.id, actor.username, confirmedAt);
+        recordProgress(id, { event: { kind: 'safety-override',
+            message: `Admin override confirmed by ${actor.username}: bypass player safety checks for this operation.`,
+            details: { actorId: actor.id, actorName: actor.username, confirmedAt, checks: ['player-occupancy'] } } });
+        return { ok: true as const };
+    })();
+}