Fix native provisioning scheduling and progress reporting

src/lib/server/provisioning/agent-events.ts

@@ -0,0 +1,27 @@
+import { db } from '../db/index.js';
+import { recordProgress } from './progress.js';
+
+/** Record receipts in the report transaction, independently of the execution worker. */
+export function recordAgentReceipt(operationId: string, state: string, at: string): void {
+    // Host operations also serve installations that have no provisioning schema yet.
+    if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_provisioning'").get()) return;
+    const job = db().prepare(`SELECT d.id, d.phase_key, n.operation_id FROM native_provisioning n
+        JOIN server_deployments d ON d.id=n.deployment_id
+        WHERE (n.operation_id=? OR n.stage_operation_id=?) AND d.server_deleted_at IS NULL
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(operationId, operationId) as
+        { id: string; phase_key: string | null; operation_id: string | null } | undefined;
+    if (!job) return;
+    const staging = job.operation_id !== operationId;
+    const message = `${staging ? 'File staging' : 'Host agent command'} ${state} (${operationId})`;
+    if (!staging && job.phase_key === 'container-action') {
+        db().prepare('UPDATE server_deployments SET stage=? WHERE id=?').run(`Host agent command ${state}`, job.id);
+    }
+    recordProgress(job.id, {
+        blocker: null,
+        event: { kind: 'agent', message, details: { operationId, state, receivedAt: at } },
+        ...(['accepted', 'running', 'completed'].includes(state) ? {
+            checkpoint: { key: `${staging ? 'staging' : 'agent'}-${state}`, label: message, at },
+            ...(!staging ? { command: { state: 'accepted' as const, message: 'Host agent acknowledged the command.', at } } : {})
+        } : {})
+    });
+}

src/lib/server/provisioning/history.ts

@@ -7,6 +7,7 @@ import { fleetSettings } from '../settings.js';
 import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
 import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
 import { bindNativeOperation } from './native.js';
+import { workerActivity } from '../pipeline/activity.js';
 
 export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
     const id = randomUUID();
@@ -101,6 +102,9 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             (ProvisioningDeployment & { observation_json: string | null; phase_key: string | null; phase_started_at: string | null; observed_at: string | null }) | undefined;
         if (!row) return null;
         const generatedAt = new Date().toISOString();
+        const native = db().prepare('SELECT instance_id FROM native_provisioning WHERE deployment_id=?').get(id);
+        const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
+            FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
         const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
         const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
             .all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
@@ -109,7 +113,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             SELECT collection_id, deployment_id FROM server_collection_members
             UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
             JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
-        return { deployment, events: deploymentEvents(id, options.events === 'recent'),
+        return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null,
+            workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
             recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
             canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),

src/lib/server/provisioning/native.ts

@@ -10,7 +10,8 @@ import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
 import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
 import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
-import { recordProgress } from './progress.js';
+import { recordProgress, type CheckInput } from './progress.js';
+import { trackActivity } from '../pipeline/activity.js';
 import { fleetSettings } from '../settings.js';
 import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
@@ -126,7 +127,7 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
                 .run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
             db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
             if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id);
-            deploymentStage(id, 'Queued for native host deployment', { phase: 'startup-queue', recipeStatus: 'queued' });
+            deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
             return { ok: true as const, name, port, deploymentId: id };
         })();
     } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
@@ -230,6 +231,10 @@ function createPayload(instance: NativeInstance, recipe: LaunchRecipe) {
 }
 
 function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
+function stillActive(job: Job): boolean {
+    return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
+        AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(job.id);
+}
 function complete(job: Job) {
     db().transaction(() => {
         db().prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
@@ -238,7 +243,8 @@ function complete(job: Job) {
     })();
 }
 
-export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null): Promise<void> {
+export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null,
+    options: { wait?: boolean } = {}): Promise<void> {
     // Draining prevents admission immediately and retires existing lobbies only through the normal safety queue.
     for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) {
         const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
@@ -253,12 +259,23 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
     const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at
         FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
         WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
-    for (const job of jobs) {
-        try { await processJob(job, eligibility); }
-        catch (error) { hold(job, (error as Error).message); }
-    }
+    // One host's observation must not prevent another job being admitted.
+    // Per-job ownership also covers overlapping direct calls and worker passes.
+    const work = jobs.map(job => {
+        const pending = activeJobs.get(job.id);
+        if (pending) return pending;
+        const work = trackActivity(`provisioning:${job.id}`, `Evaluating ${job.action} for ${job.server}`,
+            () => processJob(job, eligibility))
+            .catch(error => { if (stillActive(job)) hold(job, (error as Error).message); })
+            .finally(() => { activeJobs.delete(job.id); });
+        activeJobs.set(job.id, work);
+        return work;
+    });
+    if (options.wait !== false) await Promise.all(work);
 }
 
+const activeJobs = new Map<string, Promise<void>>();
+
 async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     let instance = nativeInstance(job.server);
     if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
@@ -289,10 +306,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         }
         if (job.action === 'destroy') {
             db().transaction(() => {
-                deploymentStage(job.id, REMOVING_SERVER, { checkpoint: { key: 'delete-deployment', label: 'Agent confirmed instance and container removal' } });
+                deploymentStage(job.id, REMOVING_SERVER, { phase: 'cleanup', checkpoint: { key: 'container-action', label: 'Agent confirmed instance and container removal', at: operation.updatedAt } });
                 db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance!.id);
                 releaseHostPorts(instance!.id);
                 db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
+                recordProgress(job.id, { checkpoint: { key: 'cleanup', label: 'Instance retired and port reservations released' } });
                 complete(job);
             })();
             return;
@@ -304,28 +322,56 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
         if (job.state !== 'awaiting-health') {
             db().prepare("UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?").run(job.id);
-            recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` }, phase: 'verify-container' });
+            recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` },
+                checkpoint: { key: 'container-action', label: `Agent confirmed ${job.action}`, at: operation.updatedAt } });
         }
+        const checks: CheckInput[] = [];
+        const check = (value: CheckInput, phase: import('../../provisioning-detail.js').ProvisioningPhase) => {
+            checks.push({ ...value, checkedAt: new Date().toISOString() });
+            recordProgress(job.id, { checks, ...(value.result === 'pass' ? { checkpoint: { key: phase, label: value.label } } : {}) });
+        };
+        deploymentStage(job.id, 'Checking container state with host agent', { phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', blocker: null, checks });
         const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect');
+        if (!stillActive(job)) return;
         if (job.action === 'stop') {
-            if (!['stopped','exited','created','dead'].includes(health.state)) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; }
+            const stopped = ['stopped','exited','created','dead'].includes(health.state);
+            check({ key: 'stopped', label: 'Stopped container verified', source: 'agent', result: stopped ? 'pass' : 'waiting', actual: health.state }, 'verify-stopped');
+            if (!stopped) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; }
             db().prepare("UPDATE servers SET container_state=?,last_players=NULL,last_roster=NULL WHERE name=?").run(health.state, job.server);
             complete(job); return;
         }
+        check({ key: 'container', label: 'Container readiness verified', source: 'agent', result: health.healthy ? 'pass' : 'waiting', actual: health.state }, 'verify-container');
         if (!health.healthy) { hold(job, `Waiting for container readiness: ${health.state}`); return; }
+        deploymentStage(job.id, 'Checking game status', { phase: 'verify-game', blocker: null });
         const status = await probeServerStatus(job.server, operation.updatedAt);
+        if (!stillActive(job)) return;
+        check({ key: 'game', label: 'Fresh game status verified', source: status.ok ? status.source : null,
+            result: status.ok ? 'pass' : 'unknown', actual: status.ok ? status.players : null,
+            message: status.ok ? null : status.reason }, 'verify-game');
         if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; }
         if (job.action === 'create' || job.action === 'rebuild') {
+            deploymentStage(job.id, 'Checking selected engine map', { phase: 'verify-map', blocker: null });
             const recipe = recipeForDeployment(job.id)!.recipe;
+            check({ key: 'map', label: 'Selected engine map verified', source: status.source,
+                result: status.map === recipe.expectedMap ? 'pass' : 'waiting', expected: recipe.expectedMap, actual: status.map }, 'verify-map');
             if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
+            deploymentStage(job.id, 'Checking production Plutonium release', { phase: 'verify-plutonium', blocker: null });
             await checkRelease();
-            const revision = await probeVersion(job.server, { operationId: job.id }); const target = freshRelease();
+            if (!stillActive(job)) return;
+            const target = freshRelease();
+            checks.push({ key: 'plutonium-release', label: 'Production Plutonium release', source: 'manifest',
+                result: target === null ? 'unknown' : 'pass', actual: target });
+            recordProgress(job.id, { checks });
+            deploymentStage(job.id, 'Checking running Plutonium revision', { phase: 'verify-plutonium', blocker: null });
+            const revision = await probeVersion(job.server, { operationId: job.id });
+            if (!stillActive(job)) return;
+            check({ key: 'plutonium-version', label: 'Plutonium revision verified', source: 'rcon', expected: target, actual: revision,
+                result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass' }, 'verify-plutonium');
             if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
             if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
         }
-        db().prepare("UPDATE servers SET container_state='running',container_checked_at=? WHERE name=?").run(new Date().toISOString(), job.server);
-        recordProgress(job.id, { blocker: null, checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', actual: health.state },
-            { key: 'game', label: 'Game status', source: status.source, result: 'pass', actual: status.players }] });
+        db().prepare('UPDATE servers SET container_state=?,container_checked_at=? WHERE name=?').run(health.state, new Date().toISOString(), job.server);
+        recordProgress(job.id, { blocker: null, checks });
         complete(job); return;
     }
     const reason = eligibility(job);
@@ -350,16 +396,23 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     const settings = fleetSettings();
     const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
         AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
-    if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
+    if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
+        deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue',
+            blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
+    }
     if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
+    deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
     if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 });
     if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
     if (job.action !== 'create' && job.action !== 'start') {
         // Fast controller check keeps occupied lobbies out of the operation slots.
         // The agent repeats this immediately before its local mutation.
+        deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
         const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
+        if (!stillActive(job)) return;
         if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
             const occupancy = await probePlayerEvidence(job.server);
+            if (!stillActive(job)) return;
             const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s
                 JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
                 WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
@@ -379,12 +432,20 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
             finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
         }
+        // Async occupancy checks can overlap another job claiming the final slot.
+        if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
+            deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
+        }
+        if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
         if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
             db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
                 DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
         }
+        recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
         let payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
         if (job.action === 'create' || job.action === 'rebuild') {
+            deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
+                checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
             const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
             const recipe: NativeRecipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image };
             payload = createPayload(instance!, recipe);
@@ -394,13 +455,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
                     recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
             for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
+            recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
         }
         const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload,
             { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id });
         db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
         for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
         db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
-        deploymentStage(job.id, `Host agent ${job.action} queued`, { phase: 'container-action', blocker: null,
+        deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null,
             command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
     })();
 }

src/lib/server/provisioning/progress.ts

@@ -36,7 +36,7 @@ export function readProgress(row: { observation_json?: string | null }): Provisi
             || (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null;
         if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(String(c.key))
             && typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result))
-            && (c.source === null || ['komodo', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
+            && (c.source === null || ['komodo', 'agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
             && typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null;
         if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string'
             && typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null;

src/lib/server/provisioning/worker.ts

@@ -14,6 +14,7 @@ import type { ProvisioningBlocker } from '../../provisioning-detail.js';
 import { recordProgress } from './progress.js';
 import { nativeInstance, nativeDeployment } from '../backends/native.js';
 import { processNativeProvisioning } from './native.js';
+import { trackActivity } from '../pipeline/activity.js';
 
 const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
 export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
@@ -91,16 +92,26 @@ function blocker(server: string, id: string, action: RuntimeAction): Provisionin
 
 interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
 let ticking: Promise<void> | null = null;
-export function processProvisioning(): Promise<void> {
-    if (ticking) return ticking;
-    ticking = run().finally(() => { ticking = null; });
-    return ticking;
+let awaitedPass: Promise<void> | null = null;
+export function processProvisioning(options: { wait?: boolean } = {}): Promise<void> {
+    if (options.wait !== false && awaitedPass) return awaitedPass;
+    // Native jobs own their individual flights. A slow observation must not
+    // prevent the next scheduler pass admitting newly arrived native work.
+    const native = nativePass(options);
+    if (!ticking) ticking = trackActivity('legacy-provisioning', 'Processing legacy provisioning', run)
+        .finally(() => { ticking = null; });
+    if (options.wait === false) {
+        void ticking.catch(error => console.error('[worker] legacy provisioning error:', error));
+        return native;
+    }
+    awaitedPass = Promise.all([native, ticking]).then(() => {}).finally(() => { awaitedPass = null; });
+    return awaitedPass;
 }
 
 export const processLifecycle = processProvisioning;
 
-async function run(): Promise<void> {
-    await processNativeProvisioning(operation => {
+function nativePass(options: { wait?: boolean }): Promise<void> {
+    return processNativeProvisioning(operation => {
         if (operation.action === 'create') return null;
         if (operation.action === 'destroy') {
             const other = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL
@@ -112,7 +123,10 @@ async function run(): Promise<void> {
         if (eligibility === 'cancelled') { cancelProvisioning(operation.id, 'Requester cancelled or replaced this operation'); return 'Requester cancelled this operation.'; }
         if (eligibility === 'held') return 'Waiting for the requester to resume.';
         return blocker(operation.server, operation.id, typed.action)?.message ?? null;
-    });
+    }, options);
+}
+
+async function run(): Promise<void> {
     await processCreations();
     await processDestructions();
     await refreshRestartRollout();