Add confirmed admin override for provisioning player checks

agent/ec_agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.14"
+VERSION = "1.4.15"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1332,7 +1332,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1569,16 +1569,20 @@ class Agent:
             else:
                 self.docker.call("container", "start", name)
         elif kind in ("instance.stop", "instance.restart", "instance.remove", "instance.rebuild"):
+            # The authenticated controller records explicit administrator approval
+            # for this immutable operation. Ownership and generation checks still apply.
+            override_safety = payload.get("safetyOverride") is True
             if kind == "instance.rebuild":
                 next_generation = payload.get("nextGeneration", payload.get("newGeneration"))
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
                     raise AgentError("Rebuild must use a newer immutable generation")
                 spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration")}
                 self.validate_spec(spec)
+            if not override_safety:
                 self.safe_to_disrupt(instance_id, generation)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
             if payload.get("stageId") and kind == "instance.restart":
-                before = self.activate(instance_id, generation, payload["stageId"])
+                before = self.activate(instance_id, generation, payload["stageId"], override_safety=override_safety)
             if kind in ("instance.remove", "instance.rebuild"):
                 if health["exists"]:
                     self.docker.call("container", "stop", "--time", "30", name)
@@ -1592,7 +1596,7 @@ class Agent:
                     shutil.rmtree(directory)
                 self.mark_removed(instance_id, generation)
             elif health["exists"]:
-                if kind == "instance.restart" and payload.get("stageId"):
+                if kind == "instance.restart" and payload.get("stageId") and not override_safety:
                     self.safe_to_disrupt(instance_id, generation)
                 self.docker.call("container", "stop" if kind.endswith("stop") else "restart", "--time", "30", name)
         else:
@@ -1663,7 +1667,7 @@ class Agent:
         atomic_json(marker, manifest)
         return {"stageId": stage_id, "files": len(planned), "state": "staged"}
 
-    def activate(self, instance_id, generation, stage_id):
+    def activate(self, instance_id, generation, stage_id, *, override_safety=False):
         directory = self.instance_dir(instance_id, generation)
         stage = directory / "stages" / identifier(stage_id)
         marker = stage / "manifest.json"
@@ -1681,6 +1685,7 @@ class Agent:
             before.append({"id": item.get("id"), "path": item["path"], "contentBase64": None if content is None else base64.b64encode(content).decode()})
         # Snapshot preparation may take time. Occupancy is rechecked at the
         # local activation boundary, after all file I/O and integrity checks.
+        if not override_safety:
             self.safe_to_disrupt(instance_id, generation)
         saved.update(state="activating", before=before)
         atomic_json(marker, saved)

agent/tests/test_agent.py

@@ -830,6 +830,64 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(self.agent.execute(self.operation("instance.stop"))["state"], "accepted")
         self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls))
 
+    def assert_override_disruption(self, kind):
+        spec = self.create()
+        payload = {"safetyOverride": True}
+        if kind == "instance.rebuild":
+            payload.update({**spec, "nextGeneration": 2})
+        operation = self.operation(kind, payload)
+        with patch.object(self.agent, "rcon", side_effect=AssertionError("Override queried player status")):
+            result = self.agent.execute(operation)
+            self.assertEqual(result["state"], "completed", result)
+            calls = list(self.docker.calls)
+            self.assertEqual(self.agent.execute(operation)["state"], "completed")
+            self.assertEqual(self.docker.calls, calls)
+        self.assertTrue((self.library / "content/game.ff").exists())
+
+    def test_admin_override_restarts_without_player_status(self):
+        self.assert_override_disruption("instance.restart")
+
+    def test_admin_override_stops_without_player_status(self):
+        self.assert_override_disruption("instance.stop")
+
+    def test_admin_override_removes_without_player_status(self):
+        self.assert_override_disruption("instance.remove")
+
+    def test_admin_override_rebuilds_without_player_status(self):
+        self.assert_override_disruption("instance.rebuild")
+
+    def test_admin_override_requires_boolean_true(self):
+        self.create()
+        self.assertIn("instance.safety-override", self.agent.inventory()["capabilities"])
+        for index, value in enumerate((False, None, "true", 1, {})):
+            with self.subTest(value=value):
+                self.agent.rcon = lambda *_: {"ok": True, "players": 2}
+                result = self.agent.execute(self.operation("instance.restart", {"safetyOverride": value}, f"attempt-{index}"))
+                self.assertEqual(result["state"], "accepted", result)
+        self.assertFalse(any(call[:2] == ("container", "restart") for call in self.docker.calls))
+
+    def test_admin_override_preserves_ownership_and_generation_checks(self):
+        self.create()
+        self.docker.containers[self.agent.container_name(INSTANCE, 1)]["Config"]["Labels"] = {}
+        self.assertEqual(self.agent.execute(self.operation("instance.remove", {"safetyOverride": True}))["state"], "failed")
+        self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls))
+        self.assertEqual(self.agent.execute(self.operation("instance.stop", {"safetyOverride": True}, "stale", generation=0))["state"], "failed")
+
+    def test_admin_override_activation_keeps_file_validation(self):
+        self.create()
+        target = self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg"
+        staged = {"stageId": "campaign", "files": [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"new").decode(), "checkBefore": True, "expectedBeforeSha": module.digest(b"original")}]}
+        self.agent.execute(self.operation("files.stage", staged, "stage"))
+        self.agent.rcon = lambda *_: {"ok": True, "players": 3}
+        target.write_bytes(b"changed")
+        failed = self.agent.execute(self.operation("instance.restart", {"stageId": "campaign", "safetyOverride": True}, "changed"))
+        self.assertEqual(failed["state"], "failed", failed)
+        self.assertFalse(any(call[:2] == ("container", "restart") for call in self.docker.calls))
+        target.write_bytes(b"original")
+        result = self.agent.execute(self.operation("instance.restart", {"stageId": "campaign", "safetyOverride": True}, "confirmed"))
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(target.read_bytes(), b"new")
+
     def test_files_stage_is_nondisruptive_activation_snapshots_and_rollback(self):
         self.create()
         target = self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg"

docs/provisioning.md

@@ -62,6 +62,17 @@ campaign, beta, maintenance, or provisioning work. Duplicate pending requests fo
 the same action return the existing operation; a different action returns a
 conflict. Requests and events survive application restarts.
 
+For a failed container stuck waiting for player status, an administrator can open
+its queued or waiting **Restart**, **Stop**, **Rebuild**, or **Destroy** operation
+and choose **Admin override…**. The confirmation modal explains the interruption
+and deletion risks. Confirming bypasses player-count and empty-server checks in
+both the controller and host agent for that operation only. The approval persists
+across restarts and records the administrator and time in the operation history.
+The host agent must support overrides (version 1.4.15 or newer); older agents show
+an update requirement. Submitted commands, host migrations, and completed operations
+cannot be overridden. Capacity, conflicting work, instance identity, file validation,
+and post-action health checks still apply.
+
 Campaigns, beta runs, and maintenance submit durable restart requests to
 Provisioning and observe the resulting operation by ID. Their pages link to that
 operation, and its history links back to the requesting work. Provisioning owns

src/lib/provisioning-detail.ts

@@ -95,6 +95,9 @@ export interface ProvisioningCapacity {
     }[];
 }
 export interface ProvisioningDetail {
+    canOverrideSafety?: boolean;
+    safetyOverrideUnavailableReason?: string | null;
+    safetyOverride?: { actorId: string; actorName: string; confirmedAt: string } | null;
     canCancel?: boolean;
     canCancelCreation?: boolean;
     cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null;

src/lib/server/auth/access.ts

@@ -29,6 +29,7 @@ export function routePermission(routeId: string | null, method: string): RoutePe
         || within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin';
 
     if (!reading && ['/api/hosts', '/api/settings', '/api/game-files'].some(base => within(routeId, base))) return 'admin';
+    if (routeId === '/api/provisioning/[id]/override') return 'admin';
     // These endpoints mix operational actions with global policy and schedule
     // changes. Per-incident/job actions remain available to operators.
     if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) return 'admin';

src/lib/server/backends/schema.ts

@@ -23,6 +23,10 @@ export function migrateNativeInstances(handle: Database.Database): void {
             applied_at TEXT, verified_at TEXT
         );
         CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id);
+        CREATE TABLE IF NOT EXISTS provisioning_safety_overrides (
+            deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
+            actor_id TEXT NOT NULL, actor_name TEXT NOT NULL, confirmed_at TEXT NOT NULL
+        );
         CREATE TABLE IF NOT EXISTS ec_host_migrations (
             deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
             source_host_id TEXT NOT NULL, instance_id TEXT NOT NULL REFERENCES native_instances(id),

src/lib/server/provisioning/history.ts

@@ -11,6 +11,7 @@ import { workerActivity } from '../pipeline/activity.js';
 import { canCancelCreation } from './cancel-creation.js';
 import { canCancelProvisioningJob, provisioningCancelled } from './cancellation.js';
 import { nativeDeployment } from '../backends/native.js';
+import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
 import { serverTitle } from '../../ui/server-labels.js';
 import { provisioningPageSize, provisioningPageOffset, type ProvisioningListOptions, type ProvisioningListPage, type ProvisioningListRow } from '../../provisioning-list.js';
 
@@ -121,7 +122,9 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             SELECT collection_id, deployment_id FROM server_collection_members
             UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
             JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
+        const override = safetyOverrideAvailability(id);
         return { deployment, backend: 'native' as const, agentOperation: agentOperation ?? null,
+            canOverrideSafety: override.applicable, safetyOverrideUnavailableReason: override.unavailableReason, safetyOverride: safetyOverride(id),
             canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
             workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,

src/lib/server/provisioning/launch-preparation.ts

@@ -14,6 +14,7 @@ export interface NativeLaunchInput {
     serverKey: string;
     rconPassword: string;
     rebuild: boolean;
+    safetyOverride?: boolean;
 }
 
 export interface PreparedNativeLaunch {

src/lib/server/provisioning/launch-worker.ts

@@ -30,7 +30,7 @@ parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeL
                 PLUTO_SERVER_KEY: input.serverKey },
             files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path),
                 contentBase64: file.content, sha256: file.sha256 })),
-            ...(input.rebuild ? { nextGeneration: instance.generation + 1 } : {})
+            ...(input.rebuild ? { nextGeneration: instance.generation + 1, ...(input.safetyOverride ? { safetyOverride: true } : {}) } : {})
         };
         const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value));
         const recipeJson = encode(recipe);

src/lib/server/provisioning/native.ts

@@ -30,6 +30,7 @@ import type { ProvisioningAction } from '../../provisioning.js';
 import { hostAgentUpdatesEnabled } from '../../hosts.js';
 import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
 import { hostMigration, processMigrationRemoval, reconcileHostDrains } from './host-drain.js';
+import { safetyOverride, safetyOverrideCapability } from './safety-override.js';
 
 export type NativeSelection = import('../../hosts.js').NativePlacement;
 type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
@@ -565,6 +566,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     const reason = eligibility(job);
     if (reason) { hold(job, reason); return; }
+    const overrideSafety = !!safetyOverride(job.id);
+    if (overrideSafety && !host.inventory.capabilities?.includes(safetyOverrideCapability)) {
+        hold(job, 'Update this host’s agent before using admin override.'); return;
+    }
     if (job.action === 'destroy' || job.action === 'rebuild') {
         const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; }
     }
@@ -596,6 +601,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
     const checkOccupancy = async (): Promise<boolean> => {
         if (!stillActive(job)) return false;
+        if (overrideSafety) return true;
         // Fast controller check keeps occupied lobbies out of the operation slots.
         // The agent repeats this immediately before its local mutation.
         deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
@@ -630,7 +636,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
             prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
                 instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
-                rebuild: job.action === 'rebuild' && !migration });
+                rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety });
             if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
         }
         const commit = db().transaction(() => {
@@ -640,6 +646,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
             const liveHost = getHost(instance!.host_id);
             if (!liveHost?.connected || liveHost.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; }
+            // Confirmation can arrive while preflight or file preparation is awaiting I/O.
+            if (!!safetyOverride(job.id) !== overrideSafety) return;
+            if (overrideSafety && !liveHost.inventory.capabilities?.includes(safetyOverrideCapability)) {
+                hold(job, 'Update this host’s agent before using admin override.'); return;
+            }
             if (job.action === 'destroy' || job.action === 'rebuild') {
                 const beta = activeBetaRemovalBlocker(job.server);
                 if (beta) { hold(job, beta); return; }
@@ -665,8 +676,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
                     DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
             }
-            recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
+            recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: overrideSafety
+                ? 'Host eligibility checked; player safety checks bypassed by admin override' : 'Host eligibility and safety checks passed' } });
             const payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
+            if (overrideSafety) payload.safetyOverride = true;
             if (prepared) {
                 deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
                     checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });

src/lib/server/provisioning/safety-override.ts

@@ -0,0 +1,44 @@
+import { db } from '../db/index.js';
+import { getHost } from '../hosts/index.js';
+import { recordProgress } from './progress.js';
+import type { ProvisioningDetail } from '../../provisioning-detail.js';
+
+export const safetyOverrideCapability = 'instance.safety-override';
+
+export function safetyOverride(id: string): ProvisioningDetail['safetyOverride'] {
+    return db().prepare(`SELECT actor_id AS actorId, actor_name AS actorName, confirmed_at AS confirmedAt
+        FROM provisioning_safety_overrides WHERE deployment_id=?`).get(id) as ProvisioningDetail['safetyOverride'] ?? null;
+}
+
+/** Approval belongs to one unsubmitted operation and its original live instance. */
+export function safetyOverrideAvailability(id: string): { applicable: boolean; unavailableReason: string | null } {
+    const target = db().prepare(`SELECT i.host_id FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.generation=n.generation AND i.retired_at IS NULL
+        WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action IN ('restart','stop','rebuild','destroy')
+        AND d.state IN ('queued','waiting') AND d.submitted_at IS NULL AND n.operation_id IS NULL
+        AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id)
+        AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.cleanup_id=d.id)
+        AND NOT EXISTS (SELECT 1 FROM provisioning_safety_overrides o WHERE o.deployment_id=d.id)`)
+        .get(id) as { host_id: string } | undefined;
+    if (!target) return { applicable: false, unavailableReason: null };
+    return { applicable: true, unavailableReason: getHost(target.host_id)?.inventory.capabilities?.includes(safetyOverrideCapability)
+        ? null : 'Update this host’s agent before using admin override.' };
+}
+
+export function enableSafetyOverride(id: string, actor: { id: string; username: string }): { ok: true } | { ok: false; error: string } {
+    return db().transaction(() => {
+        // A retried confirmation never sends or repeats a command.
+        if (safetyOverride(id)) return { ok: true as const };
+        const availability = safetyOverrideAvailability(id);
+        if (!availability.applicable) return { ok: false as const, error: 'This operation can no longer be overridden. Refresh to see its current state.' };
+        if (availability.unavailableReason) return { ok: false as const, error: availability.unavailableReason };
+        const confirmedAt = new Date().toISOString();
+        db().prepare('INSERT INTO provisioning_safety_overrides(deployment_id,actor_id,actor_name,confirmed_at) VALUES(?,?,?,?)')
+            .run(id, actor.id, actor.username, confirmedAt);
+        recordProgress(id, { event: { kind: 'safety-override',
+            message: `Admin override confirmed by ${actor.username}: bypass player safety checks for this operation.`,
+            details: { actorId: actor.id, actorName: actor.username, confirmedAt, checks: ['player-occupancy'] } } });
+        return { ok: true as const };
+    })();
+}

src/lib/ui/provisioning-refresh.ts

@@ -1,4 +1,4 @@
-export type ProvisioningMutation = 'resume' | 'cancel' | 'cancel-creation';
+export type ProvisioningMutation = 'resume' | 'cancel' | 'cancel-creation' | 'override';
 
 export interface ProvisioningRefreshState<T> {
     id: string;
@@ -88,7 +88,7 @@ export function createProvisioningRefresh<T>(options: Options<T>) {
         if (!detail) void load();
     }
 
-    async function act(action: ProvisioningMutation, successMessage: string): Promise<void> {
+    async function act(action: ProvisioningMutation, successMessage: string, payload?: Record<string, unknown>): Promise<void> {
         if (disposed || !state.id || state.busy) return;
         const currentVisit = visit;
         const id = state.id;
@@ -100,7 +100,8 @@ export function createProvisioningRefresh<T>(options: Options<T>) {
         const timer = setTimeout(() => { timedOut = true; controller.abort(); }, options.timeoutMs ?? 15_000);
         publish({ busy: action, refreshing: false, actionError: '', notice: '' });
         try {
-            const response = await request(`/api/provisioning/${encodeURIComponent(id)}/${action}`, { method: 'POST', signal: controller.signal });
+            const response = await request(`/api/provisioning/${encodeURIComponent(id)}/${action}`, { method: 'POST', signal: controller.signal,
+                ...(payload ? { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) } : {}) });
             const body = await response.json() as { error?: string };
             if (!response.ok) throw new Error(body.error ?? 'The operation changed before this request could be accepted.');
             if (current()) publish({ notice: successMessage });

src/routes/api/provisioning/[id]/override/+server.ts

@@ -0,0 +1,16 @@
+import { json } from '@sveltejs/kit';
+import { authBody, authError } from '$lib/server/auth/http.js';
+import { enableSafetyOverride } from '$lib/server/provisioning/safety-override.js';
+import { kickProvisioning } from '$lib/server/pipeline/worker.js';
+import type { RequestHandler } from './$types';
+
+export const POST: RequestHandler = async ({ params, request, locals }) => {
+    if (!locals.user || locals.user.role !== 'admin') return json({ error: 'Administrator access is required.' }, { status: 403 });
+    let body: Record<string, unknown>;
+    try { body = await authBody(request); } catch (error) { return authError(error); }
+    if (body.confirmed !== true) return json({ error: 'Confirm the admin override before continuing.' }, { status: 400 });
+    const result = enableSafetyOverride(params.id, locals.user);
+    if (!result.ok) return json({ error: result.error }, { status: 409 });
+    void kickProvisioning();
+    return json({ ok: true }, { status: 202 });
+};

src/routes/provisioning/[id]/+page.svelte

@@ -8,6 +8,7 @@
     import { pollVisible } from '$ui/poll.js';
     import { serverTitle } from '$ui/server-labels.js';
     import { createProvisioningRefresh, type ProvisioningRefreshState } from '$ui/provisioning-refresh.js';
+    import { confirmAction } from '$ui/confirm.js';
     import { elapsedTime, isProvisioningDetail, provisioningOutcome, provisioningReturn } from '$ui/provisioning-presentation.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
@@ -33,6 +34,7 @@
     const controller = createProvisioningRefresh<ProvisioningDetail>({ validate: isProvisioningDetail, onChange: value => { view = value; } });
     const detail = $derived(view.detail);
     const deployment = $derived(detail?.deployment);
+    const canOverride = $derived(data.user?.role === 'admin' && detail?.canOverrideSafety);
     const outcome = $derived(detail ? provisioningOutcome(detail) : null);
     const returnTo = $derived.by(() => {
         const requested = $page.url.searchParams.get('returnTo');
@@ -56,6 +58,19 @@
     onMount(() => pollVisible(() => { now = Date.now(); return controller.refresh(); }, 5000, false));
     onDestroy(() => controller.dispose());
     function fmt(value: string | null | undefined) { return value ? $timeFormat.dateTime(value) : 'Not recorded'; }
+    async function overrideSafety() {
+        if (!deployment || !canOverride || view.busy || detail?.safetyOverrideUnavailableReason) return;
+        const id = deployment.id;
+        const action = actionLabels[deployment.action];
+        const destructive = ['rebuild', 'destroy'].includes(deployment.action);
+        const confirmed = await confirmAction({
+            title: `Admin override: ${action.toLowerCase()} server?`,
+            message: `${action} ${serverTitle(deployment.server)} (${deployment.server}) without waiting for a fresh player count or an empty server. This can disconnect active players and interrupt their game.${destructive ? '\n\nThis action deletes the container and all instance files.' : ''}\n\nThis override applies only to this operation and records your administrator identity. Capacity, operation conflicts, file validation, and post-action verification still apply.`,
+            confirmLabel: `Override and ${action.toLowerCase()}`, danger: true
+        });
+        if (confirmed && view.id === id && canOverride) await controller.act('override',
+            'Admin override enabled for this operation. Provisioning will proceed without player safety checks when the remaining requirements are met.', { confirmed: true });
+    }
     async function copyLink() {
         copyStatus = '';
         try {
@@ -89,8 +104,10 @@
             {#if detail.activity.failureExcerpt && detail.activity.failureExcerpt.message !== deployment.error && ['failed', 'interrupted'].includes(deployment.state)}<p class="failure-excerpt"><span>Last failure event · {fmt(detail.activity.failureExcerpt.created_at)}</span>{detail.activity.failureExcerpt.message}</p>{/if}
             {#if detail.warningMessages.length}<div class="warnings"><h3>Recorded warnings</h3><ul>{#each detail.warningMessages as warning}<li>{warning}</li>{/each}</ul></div>{/if}
 
-            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancel || detail.canCancelCreation}
+            {#if detail.safetyOverride}<div class="warnings"><h3>Admin override enabled</h3><p>Player safety checks bypassed for this operation. Confirmed by {detail.safetyOverride.actorName} · {fmt(detail.safetyOverride.confirmedAt)}.</p></div>{/if}
+            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
                 <div class="recovery-actions">
+                    {#if canOverride}<div><Button variant="danger" disabled={!!view.busy || !!detail.safetyOverrideUnavailableReason} loading={view.busy === 'override'} onclick={overrideSafety}>Admin override…</Button><p class="action-help">{detail.safetyOverrideUnavailableReason ?? 'Confirm to bypass player safety checks for this operation.'}</p></div>{/if}
                     {#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. The existing instance will be observed without another creation or restart.')}>Resume health verification</Button>{/if}
                     {#if detail.canResumeRebuild}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Rebuild cleanup requested. Fresh creation waits for local cleanup to finish.')}>Resume rebuild cleanup</Button>{/if}
                     {#if detail.canResumeCleanup}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Local cleanup requested. Remote container deletion will not be repeated.')}>Resume cleanup</Button>{/if}

static/agent/ec-agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.14"
+VERSION = "1.4.15"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1332,7 +1332,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1569,16 +1569,20 @@ class Agent:
             else:
                 self.docker.call("container", "start", name)
         elif kind in ("instance.stop", "instance.restart", "instance.remove", "instance.rebuild"):
+            # The authenticated controller records explicit administrator approval
+            # for this immutable operation. Ownership and generation checks still apply.
+            override_safety = payload.get("safetyOverride") is True
             if kind == "instance.rebuild":
                 next_generation = payload.get("nextGeneration", payload.get("newGeneration"))
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
                     raise AgentError("Rebuild must use a newer immutable generation")
                 spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration")}
                 self.validate_spec(spec)
+            if not override_safety:
                 self.safe_to_disrupt(instance_id, generation)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
             if payload.get("stageId") and kind == "instance.restart":
-                before = self.activate(instance_id, generation, payload["stageId"])
+                before = self.activate(instance_id, generation, payload["stageId"], override_safety=override_safety)
             if kind in ("instance.remove", "instance.rebuild"):
                 if health["exists"]:
                     self.docker.call("container", "stop", "--time", "30", name)
@@ -1592,7 +1596,7 @@ class Agent:
                     shutil.rmtree(directory)
                 self.mark_removed(instance_id, generation)
             elif health["exists"]:
-                if kind == "instance.restart" and payload.get("stageId"):
+                if kind == "instance.restart" and payload.get("stageId") and not override_safety:
                     self.safe_to_disrupt(instance_id, generation)
                 self.docker.call("container", "stop" if kind.endswith("stop") else "restart", "--time", "30", name)
         else:
@@ -1663,7 +1667,7 @@ class Agent:
         atomic_json(marker, manifest)
         return {"stageId": stage_id, "files": len(planned), "state": "staged"}
 
-    def activate(self, instance_id, generation, stage_id):
+    def activate(self, instance_id, generation, stage_id, *, override_safety=False):
         directory = self.instance_dir(instance_id, generation)
         stage = directory / "stages" / identifier(stage_id)
         marker = stage / "manifest.json"
@@ -1681,6 +1685,7 @@ class Agent:
             before.append({"id": item.get("id"), "path": item["path"], "contentBase64": None if content is None else base64.b64encode(content).decode()})
         # Snapshot preparation may take time. Occupancy is rechecked at the
         # local activation boundary, after all file I/O and integrity checks.
+        if not override_safety:
             self.safe_to_disrupt(instance_id, generation)
         saved.update(state="activating", before=before)
         atomic_json(marker, saved)

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.4.14",
+  "version": "1.4.15",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "3e2e11389f412a0f1b2746c85b9afa626c5e0fcaa4801bd13f1de283abf8830e",
+  "sha256": "ddd1f7d557e7d4441b7cff1d8cbc4ed80aed7e78a6f1ab29656c5bdb55410543",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/auth-access.test.ts

@@ -37,7 +37,7 @@ describe('management route permissions', () => {
         expect(allows('admin', route, method)).toBe(true);
     });
 
-    it.each(['/api/users', '/api/users/[id]', '/api/users/[id]/password', '/settings', '/settings/users', '/server-keys', '/keys', '/api/server-keys'])('reserves all methods on %s for administrators', route => {
+    it.each(['/api/users', '/api/users/[id]', '/api/users/[id]/password', '/settings', '/settings/users', '/server-keys', '/keys', '/api/server-keys', '/api/provisioning/[id]/override'])('reserves all methods on %s for administrators', route => {
         for (const method of ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE']) {
             expect(routePermission(route, method)).toBe('admin');
             expect(allows('viewer', route, method)).toBe(false);

tests/unit/native-provisioning.test.ts

@@ -463,6 +463,23 @@ describe('native instance provisioning', () => {
         await advance();
         expect(nativeInstance(job.name)?.generation).toBe(2);
     });
+    it('carries confirmed admin override through serialized rebuild preparation', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        heartbeat(hostId, { ...inventory(), capabilities: ['instance.safety-override'] });
+        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
+        remote.players = 2; await advance();
+        expect(await operationFor(rebuild.deploymentId)).toBeNull();
+        const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
+        expect(enableSafetyOverride(rebuild.deploymentId, { id: 'admin', username: 'admin' })).toEqual({ ok: true });
+        await advance();
+        expect(await operationFor(rebuild.deploymentId)).toMatchObject({ kind: 'instance.rebuild', generation: 1,
+            payload: { nextGeneration: 2, libraryId, safetyOverride: true } });
+        await acknowledge(rebuild.deploymentId);
+        expect((await detail(rebuild.deploymentId)).state).toBe('done');
+    });
+
     it('holds a rebuild when players join while its launch files are being prepared', async () => {
         const job = await queue(); await advance(); await acknowledge(job.deploymentId);
         const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');

tests/unit/native-safety.test.ts

@@ -6,7 +6,7 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
     rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } })),
     rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
 }));
-vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
 
 let fixture: Fixture; let hostId: string; let instanceId: string;
 const server = 't5_zombie_theater_1';
@@ -17,7 +17,7 @@ beforeEach(async () => {
     const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js');
     hostId = createHost({ name: 'Safety test host' }).id;
     enrollHost(createEnrollment(hostId).token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
-        storageAvailable: 100_000_000_000, runtimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }] });
+        storageAvailable: 100_000_000_000, runtimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }], capabilities: ['instance.safety-override'] });
     const { db } = await import('../../src/lib/server/db/index.js');
     instanceId = randomUUID();
     db().prepare("INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'t5','zombie_theater','zombie','theater',28960,'running')").run(server);
@@ -120,6 +120,133 @@ describe('native disruption safety at dispatch', () => {
     });
 });
 
+describe('provisioning admin override', () => {
+    const actor = { id: 'admin-id', username: 'recovery-admin' };
+    async function confirm(id: string) {
+        const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
+        expect(enableSafetyOverride(id, actor)).toEqual({ ok: true });
+    }
+
+    it.each(['restart', 'stop', 'destroy'] as const)('dispatches %s with unavailable player evidence only after confirmation', async action => {
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
+        const result = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, action);
+        if (!result.ok || !result.deploymentId) throw new Error('Request failed');
+        const id = result.deploymentId;
+        const status = await import('../../src/lib/server/fleet/status-evidence.js');
+        vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ players: null, source: null, observedAt: new Date().toISOString() });
+        await advance();
+        expect(await operationId(id)).toBeNull();
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        expect(getDeployment(id)).toMatchObject({ canOverrideSafety: true, safetyOverride: null });
+        await confirm(id);
+        await confirm(id);
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        closeDb(); // Approval and attribution survive controller restart.
+        await advance();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        const commandId = (await operationId(id))!;
+        expect(getHostOperation(commandId)).toMatchObject({ kind: `instance.${action === 'destroy' ? 'remove' : action}`, payload: { safetyOverride: true } });
+        expect(getDeployment(id)).toMatchObject({ canOverrideSafety: false, safetyOverride: { actorId: actor.id, actorName: actor.username } });
+        expect(db().prepare("SELECT COUNT(*) n FROM server_deployment_events WHERE deployment_id=? AND kind='safety-override'").get(id)).toEqual({ n: 1 });
+        await advance();
+        expect(await operationId(id)).toBe(commandId);
+    });
+
+    it('bypasses occupied telemetry while retaining health verification after the restart', async () => {
+        const id = await waitingRestart();
+        const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
+        saveTelemetry({ event: 'game_start', server, map: 'zombie_theater', players: [{ name: 'Playing', guid: '1' }], receivedAt: new Date().toISOString() });
+        await advance();
+        expect(await operationId(id)).toBeNull();
+        await confirm(id);
+        await advance();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, (await operationId(id))!, { state: 'completed', result: {} });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockResolvedValue({ state: 'running', healthy: false } as never);
+        await advance();
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        expect(getDeployment(id)?.deployment.state).toBe('awaiting-health');
+        expect(getDeployment(id)?.progress?.checks).toContainEqual(expect.objectContaining({ key: 'container', result: 'waiting' }));
+    });
+
+    it('retains recovery capacity limits and cancellation after approval', async () => {
+        const id = await waitingRestart();
+        await confirm(id);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 });
+        db().prepare("INSERT INTO servers(name) VALUES('other')").run();
+        db().prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('other',?)").run(new Date().toISOString());
+        await advance();
+        expect(await operationId(id)).toBeNull();
+        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
+        expect(cancelProvisioningJob(id)).toBe(true);
+        db().prepare("DELETE FROM restart_rollout WHERE server='other'").run();
+        await advance();
+        expect(await operationId(id)).toBeNull();
+    });
+
+    it('rejects old agents, stale instances, unsupported actions and submitted or terminal operations', async () => {
+        const id = await waitingRestart();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { enableSafetyOverride, safetyOverrideAvailability } = await import('../../src/lib/server/provisioning/safety-override.js');
+        db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json('[]')) WHERE id=?").run(hostId);
+        expect(enableSafetyOverride(id, actor)).toMatchObject({ ok: false, error: expect.stringContaining('Update') });
+        for (const state of ['running', 'awaiting-health', 'failed', 'interrupted', 'cancelled', 'done']) {
+            db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, id);
+            expect(safetyOverrideAvailability(id).applicable).toBe(false);
+        }
+        db().prepare("UPDATE server_deployments SET state='waiting',action='start' WHERE id=?").run(id);
+        expect(safetyOverrideAvailability(id).applicable).toBe(false);
+        db().prepare("UPDATE server_deployments SET action='restart',submitted_at='2026-10-05' WHERE id=?").run(id);
+        expect(safetyOverrideAvailability(id).applicable).toBe(false);
+        db().prepare('UPDATE server_deployments SET submitted_at=NULL WHERE id=?').run(id);
+        db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId);
+        expect(safetyOverrideAvailability(id).applicable).toBe(false);
+        expect(enableSafetyOverride('missing', actor).ok).toBe(false);
+    });
+
+    it('rechecks approval after an in-flight inspection before dispatch', async () => {
+        const id = await waitingRestart();
+        const pending = await suspendInspection();
+        await confirm(id);
+        pending.release(); await pending.running;
+        expect(await operationId(id)).toBeNull();
+        await advance();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ safetyOverride: true });
+    });
+
+    it('overrides a waiting campaign restart while keeping its staged changes attached', async () => {
+        const id = await stageCampaign();
+        const status = await import('../../src/lib/server/fleet/status-evidence.js');
+        vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ players: null, source: null, observedAt: new Date().toISOString() });
+        await advance();
+        expect(await operationId(id)).toBeNull();
+        await confirm(id);
+        await advance();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ stageId: id, safetyOverride: true });
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_id='campaign-race'").get()).toEqual({ state: 'applying' });
+    });
+
+    it('requires an administrator and explicit confirmation at the API boundary', async () => {
+        const id = await waitingRestart();
+        const { POST } = await import('../../src/routes/api/provisioning/[id]/override/+server.js');
+        const request = (role: string | null, confirmed: unknown) => POST({ params: { id }, locals: { user: role ? { ...actor, role } : null },
+            request: new Request(`http://localhost/api/provisioning/${id}/override`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ confirmed }) })
+        } as Parameters<typeof POST>[0]);
+        for (const role of [null, 'viewer', 'operator']) expect((await request(role, true)).status).toBe(403);
+        for (const value of [undefined, false, 'true', 1]) expect((await request('admin', value)).status).toBe(400);
+        expect((await request('admin', true)).status).toBe(202);
+        const { safetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
+        expect(safetyOverride(id)?.actorId).toBe(actor.id);
+    });
+});
+
 describe('native scheduler ownership', () => {
     it('rejects lifecycle commands for a registry row without a host instance', async () => {
         const { db } = await import('../../src/lib/server/db/index.js');

tests/unit/provisioning-refresh.test.ts

@@ -23,6 +23,16 @@ function deferred() {
 }
 
 describe('provisioning detail refresh ownership', () => {
+    it('sends explicit override confirmation and refreshes the operation after acceptance', async () => {
+        const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true })).mockResolvedValueOnce(response('a', 2));
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'));
+        await page.act('override', 'Override enabled.', { confirmed: true });
+        expect(fetcher).toHaveBeenNthCalledWith(1, '/api/provisioning/a/override', expect.objectContaining({
+            method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{"confirmed":true}'
+        }));
+        expect(page.state()).toMatchObject({ detail: detail('a', 2), notice: 'Override enabled.', busy: null });
+    });
     it('uses the server snapshot without another initial request and coalesces background reads', async () => {
         const pending = deferred();
         const fetcher = vi.fn(() => pending.promise);