Add confirmed admin override for provisioning player checks

src/lib/provisioning-detail.ts

@@ -95,6 +95,9 @@ export interface ProvisioningCapacity {
     }[];
 }
 export interface ProvisioningDetail {
+    canOverrideSafety?: boolean;
+    safetyOverrideUnavailableReason?: string | null;
+    safetyOverride?: { actorId: string; actorName: string; confirmedAt: string } | null;
     canCancel?: boolean;
     canCancelCreation?: boolean;
     cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null;

src/lib/server/auth/access.ts

@@ -29,6 +29,7 @@ export function routePermission(routeId: string | null, method: string): RoutePe
         || within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin';
 
     if (!reading && ['/api/hosts', '/api/settings', '/api/game-files'].some(base => within(routeId, base))) return 'admin';
+    if (routeId === '/api/provisioning/[id]/override') return 'admin';
     // These endpoints mix operational actions with global policy and schedule
     // changes. Per-incident/job actions remain available to operators.
     if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) return 'admin';

src/lib/server/backends/schema.ts

@@ -23,6 +23,10 @@ export function migrateNativeInstances(handle: Database.Database): void {
             applied_at TEXT, verified_at TEXT
         );
         CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id);
+        CREATE TABLE IF NOT EXISTS provisioning_safety_overrides (
+            deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
+            actor_id TEXT NOT NULL, actor_name TEXT NOT NULL, confirmed_at TEXT NOT NULL
+        );
         CREATE TABLE IF NOT EXISTS ec_host_migrations (
             deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
             source_host_id TEXT NOT NULL, instance_id TEXT NOT NULL REFERENCES native_instances(id),

src/lib/server/provisioning/history.ts

@@ -11,6 +11,7 @@ import { workerActivity } from '../pipeline/activity.js';
 import { canCancelCreation } from './cancel-creation.js';
 import { canCancelProvisioningJob, provisioningCancelled } from './cancellation.js';
 import { nativeDeployment } from '../backends/native.js';
+import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
 import { serverTitle } from '../../ui/server-labels.js';
 import { provisioningPageSize, provisioningPageOffset, type ProvisioningListOptions, type ProvisioningListPage, type ProvisioningListRow } from '../../provisioning-list.js';
 
@@ -121,7 +122,9 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             SELECT collection_id, deployment_id FROM server_collection_members
             UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
             JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
+        const override = safetyOverrideAvailability(id);
         return { deployment, backend: 'native' as const, agentOperation: agentOperation ?? null,
+            canOverrideSafety: override.applicable, safetyOverrideUnavailableReason: override.unavailableReason, safetyOverride: safetyOverride(id),
             canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
             workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,

src/lib/server/provisioning/launch-preparation.ts

@@ -14,6 +14,7 @@ export interface NativeLaunchInput {
     serverKey: string;
     rconPassword: string;
     rebuild: boolean;
+    safetyOverride?: boolean;
 }
 
 export interface PreparedNativeLaunch {

src/lib/server/provisioning/launch-worker.ts

@@ -30,7 +30,7 @@ parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeL
                 PLUTO_SERVER_KEY: input.serverKey },
             files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path),
                 contentBase64: file.content, sha256: file.sha256 })),
-            ...(input.rebuild ? { nextGeneration: instance.generation + 1 } : {})
+            ...(input.rebuild ? { nextGeneration: instance.generation + 1, ...(input.safetyOverride ? { safetyOverride: true } : {}) } : {})
         };
         const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value));
         const recipeJson = encode(recipe);

src/lib/server/provisioning/native.ts

@@ -30,6 +30,7 @@ import type { ProvisioningAction } from '../../provisioning.js';
 import { hostAgentUpdatesEnabled } from '../../hosts.js';
 import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
 import { hostMigration, processMigrationRemoval, reconcileHostDrains } from './host-drain.js';
+import { safetyOverride, safetyOverrideCapability } from './safety-override.js';
 
 export type NativeSelection = import('../../hosts.js').NativePlacement;
 type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
@@ -565,6 +566,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     const reason = eligibility(job);
     if (reason) { hold(job, reason); return; }
+    const overrideSafety = !!safetyOverride(job.id);
+    if (overrideSafety && !host.inventory.capabilities?.includes(safetyOverrideCapability)) {
+        hold(job, 'Update this host’s agent before using admin override.'); return;
+    }
     if (job.action === 'destroy' || job.action === 'rebuild') {
         const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; }
     }
@@ -596,6 +601,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
     const checkOccupancy = async (): Promise<boolean> => {
         if (!stillActive(job)) return false;
+        if (overrideSafety) return true;
         // Fast controller check keeps occupied lobbies out of the operation slots.
         // The agent repeats this immediately before its local mutation.
         deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
@@ -630,7 +636,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
             prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
                 instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
-                rebuild: job.action === 'rebuild' && !migration });
+                rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety });
             if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
         }
         const commit = db().transaction(() => {
@@ -640,6 +646,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
             const liveHost = getHost(instance!.host_id);
             if (!liveHost?.connected || liveHost.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; }
+            // Confirmation can arrive while preflight or file preparation is awaiting I/O.
+            if (!!safetyOverride(job.id) !== overrideSafety) return;
+            if (overrideSafety && !liveHost.inventory.capabilities?.includes(safetyOverrideCapability)) {
+                hold(job, 'Update this host’s agent before using admin override.'); return;
+            }
             if (job.action === 'destroy' || job.action === 'rebuild') {
                 const beta = activeBetaRemovalBlocker(job.server);
                 if (beta) { hold(job, beta); return; }
@@ -665,8 +676,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
                     DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
             }
-            recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
+            recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: overrideSafety
+                ? 'Host eligibility checked; player safety checks bypassed by admin override' : 'Host eligibility and safety checks passed' } });
             const payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
+            if (overrideSafety) payload.safetyOverride = true;
             if (prepared) {
                 deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
                     checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });

src/lib/server/provisioning/safety-override.ts

@@ -0,0 +1,44 @@
+import { db } from '../db/index.js';
+import { getHost } from '../hosts/index.js';
+import { recordProgress } from './progress.js';
+import type { ProvisioningDetail } from '../../provisioning-detail.js';
+
+export const safetyOverrideCapability = 'instance.safety-override';
+
+export function safetyOverride(id: string): ProvisioningDetail['safetyOverride'] {
+    return db().prepare(`SELECT actor_id AS actorId, actor_name AS actorName, confirmed_at AS confirmedAt
+        FROM provisioning_safety_overrides WHERE deployment_id=?`).get(id) as ProvisioningDetail['safetyOverride'] ?? null;
+}
+
+/** Approval belongs to one unsubmitted operation and its original live instance. */
+export function safetyOverrideAvailability(id: string): { applicable: boolean; unavailableReason: string | null } {
+    const target = db().prepare(`SELECT i.host_id FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.generation=n.generation AND i.retired_at IS NULL
+        WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action IN ('restart','stop','rebuild','destroy')
+        AND d.state IN ('queued','waiting') AND d.submitted_at IS NULL AND n.operation_id IS NULL
+        AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id)
+        AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.cleanup_id=d.id)
+        AND NOT EXISTS (SELECT 1 FROM provisioning_safety_overrides o WHERE o.deployment_id=d.id)`)
+        .get(id) as { host_id: string } | undefined;
+    if (!target) return { applicable: false, unavailableReason: null };
+    return { applicable: true, unavailableReason: getHost(target.host_id)?.inventory.capabilities?.includes(safetyOverrideCapability)
+        ? null : 'Update this host’s agent before using admin override.' };
+}
+
+export function enableSafetyOverride(id: string, actor: { id: string; username: string }): { ok: true } | { ok: false; error: string } {
+    return db().transaction(() => {
+        // A retried confirmation never sends or repeats a command.
+        if (safetyOverride(id)) return { ok: true as const };
+        const availability = safetyOverrideAvailability(id);
+        if (!availability.applicable) return { ok: false as const, error: 'This operation can no longer be overridden. Refresh to see its current state.' };
+        if (availability.unavailableReason) return { ok: false as const, error: availability.unavailableReason };
+        const confirmedAt = new Date().toISOString();
+        db().prepare('INSERT INTO provisioning_safety_overrides(deployment_id,actor_id,actor_name,confirmed_at) VALUES(?,?,?,?)')
+            .run(id, actor.id, actor.username, confirmedAt);
+        recordProgress(id, { event: { kind: 'safety-override',
+            message: `Admin override confirmed by ${actor.username}: bypass player safety checks for this operation.`,
+            details: { actorId: actor.id, actorName: actor.username, confirmedAt, checks: ['player-occupancy'] } } });
+        return { ok: true as const };
+    })();
+}

src/lib/ui/provisioning-refresh.ts

@@ -1,4 +1,4 @@
-export type ProvisioningMutation = 'resume' | 'cancel' | 'cancel-creation';
+export type ProvisioningMutation = 'resume' | 'cancel' | 'cancel-creation' | 'override';
 
 export interface ProvisioningRefreshState<T> {
     id: string;
@@ -88,7 +88,7 @@ export function createProvisioningRefresh<T>(options: Options<T>) {
         if (!detail) void load();
     }
 
-    async function act(action: ProvisioningMutation, successMessage: string): Promise<void> {
+    async function act(action: ProvisioningMutation, successMessage: string, payload?: Record<string, unknown>): Promise<void> {
         if (disposed || !state.id || state.busy) return;
         const currentVisit = visit;
         const id = state.id;
@@ -100,7 +100,8 @@ export function createProvisioningRefresh<T>(options: Options<T>) {
         const timer = setTimeout(() => { timedOut = true; controller.abort(); }, options.timeoutMs ?? 15_000);
         publish({ busy: action, refreshing: false, actionError: '', notice: '' });
         try {
-            const response = await request(`/api/provisioning/${encodeURIComponent(id)}/${action}`, { method: 'POST', signal: controller.signal });
+            const response = await request(`/api/provisioning/${encodeURIComponent(id)}/${action}`, { method: 'POST', signal: controller.signal,
+                ...(payload ? { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) } : {}) });
             const body = await response.json() as { error?: string };
             if (!response.ok) throw new Error(body.error ?? 'The operation changed before this request could be accepted.');
             if (current()) publish({ notice: successMessage });

src/routes/api/provisioning/[id]/override/+server.ts

@@ -0,0 +1,16 @@
+import { json } from '@sveltejs/kit';
+import { authBody, authError } from '$lib/server/auth/http.js';
+import { enableSafetyOverride } from '$lib/server/provisioning/safety-override.js';
+import { kickProvisioning } from '$lib/server/pipeline/worker.js';
+import type { RequestHandler } from './$types';
+
+export const POST: RequestHandler = async ({ params, request, locals }) => {
+    if (!locals.user || locals.user.role !== 'admin') return json({ error: 'Administrator access is required.' }, { status: 403 });
+    let body: Record<string, unknown>;
+    try { body = await authBody(request); } catch (error) { return authError(error); }
+    if (body.confirmed !== true) return json({ error: 'Confirm the admin override before continuing.' }, { status: 400 });
+    const result = enableSafetyOverride(params.id, locals.user);
+    if (!result.ok) return json({ error: result.error }, { status: 409 });
+    void kickProvisioning();
+    return json({ ok: true }, { status: 202 });
+};

src/routes/provisioning/[id]/+page.svelte

@@ -8,6 +8,7 @@
     import { pollVisible } from '$ui/poll.js';
     import { serverTitle } from '$ui/server-labels.js';
     import { createProvisioningRefresh, type ProvisioningRefreshState } from '$ui/provisioning-refresh.js';
+    import { confirmAction } from '$ui/confirm.js';
     import { elapsedTime, isProvisioningDetail, provisioningOutcome, provisioningReturn } from '$ui/provisioning-presentation.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
@@ -33,6 +34,7 @@
     const controller = createProvisioningRefresh<ProvisioningDetail>({ validate: isProvisioningDetail, onChange: value => { view = value; } });
     const detail = $derived(view.detail);
     const deployment = $derived(detail?.deployment);
+    const canOverride = $derived(data.user?.role === 'admin' && detail?.canOverrideSafety);
     const outcome = $derived(detail ? provisioningOutcome(detail) : null);
     const returnTo = $derived.by(() => {
         const requested = $page.url.searchParams.get('returnTo');
@@ -56,6 +58,19 @@
     onMount(() => pollVisible(() => { now = Date.now(); return controller.refresh(); }, 5000, false));
     onDestroy(() => controller.dispose());
     function fmt(value: string | null | undefined) { return value ? $timeFormat.dateTime(value) : 'Not recorded'; }
+    async function overrideSafety() {
+        if (!deployment || !canOverride || view.busy || detail?.safetyOverrideUnavailableReason) return;
+        const id = deployment.id;
+        const action = actionLabels[deployment.action];
+        const destructive = ['rebuild', 'destroy'].includes(deployment.action);
+        const confirmed = await confirmAction({
+            title: `Admin override: ${action.toLowerCase()} server?`,
+            message: `${action} ${serverTitle(deployment.server)} (${deployment.server}) without waiting for a fresh player count or an empty server. This can disconnect active players and interrupt their game.${destructive ? '\n\nThis action deletes the container and all instance files.' : ''}\n\nThis override applies only to this operation and records your administrator identity. Capacity, operation conflicts, file validation, and post-action verification still apply.`,
+            confirmLabel: `Override and ${action.toLowerCase()}`, danger: true
+        });
+        if (confirmed && view.id === id && canOverride) await controller.act('override',
+            'Admin override enabled for this operation. Provisioning will proceed without player safety checks when the remaining requirements are met.', { confirmed: true });
+    }
     async function copyLink() {
         copyStatus = '';
         try {
@@ -89,8 +104,10 @@
             {#if detail.activity.failureExcerpt && detail.activity.failureExcerpt.message !== deployment.error && ['failed', 'interrupted'].includes(deployment.state)}<p class="failure-excerpt"><span>Last failure event · {fmt(detail.activity.failureExcerpt.created_at)}</span>{detail.activity.failureExcerpt.message}</p>{/if}
             {#if detail.warningMessages.length}<div class="warnings"><h3>Recorded warnings</h3><ul>{#each detail.warningMessages as warning}<li>{warning}</li>{/each}</ul></div>{/if}
 
-            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancel || detail.canCancelCreation}
+            {#if detail.safetyOverride}<div class="warnings"><h3>Admin override enabled</h3><p>Player safety checks bypassed for this operation. Confirmed by {detail.safetyOverride.actorName} · {fmt(detail.safetyOverride.confirmedAt)}.</p></div>{/if}
+            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
                 <div class="recovery-actions">
+                    {#if canOverride}<div><Button variant="danger" disabled={!!view.busy || !!detail.safetyOverrideUnavailableReason} loading={view.busy === 'override'} onclick={overrideSafety}>Admin override…</Button><p class="action-help">{detail.safetyOverrideUnavailableReason ?? 'Confirm to bypass player safety checks for this operation.'}</p></div>{/if}
                     {#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. The existing instance will be observed without another creation or restart.')}>Resume health verification</Button>{/if}
                     {#if detail.canResumeRebuild}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Rebuild cleanup requested. Fresh creation waits for local cleanup to finish.')}>Resume rebuild cleanup</Button>{/if}
                     {#if detail.canResumeCleanup}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Local cleanup requested. Remote container deletion will not be repeated.')}>Resume cleanup</Button>{/if}