josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix native provisioning scheduling and progress reporting

Browse Source
This commit is contained in:
josh
2026-10-04 14:50:59 -04:00
parent 1bbee98b62
commit 2adadb0d05
18 changed files with 569 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/native.ts
+78 -16
View File
@@ -10,7 +10,8 @@ import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
import { recordProgress } from './progress.js';
import { recordProgress, type CheckInput } from './progress.js';
import { trackActivity } from '../pipeline/activity.js';
import { fleetSettings } from '../settings.js';
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
@@ -126,7 +127,7 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id);
deploymentStage(id, 'Queued for native host deployment', { phase: 'startup-queue', recipeStatus: 'queued' });
deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port, deploymentId: id };
})();
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
@@ -230,6 +231,10 @@ function createPayload(instance: NativeInstance, recipe: LaunchRecipe) {
}
function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
function stillActive(job: Job): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(job.id);
}
function complete(job: Job) {
db().transaction(() => {
db().prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
@@ -238,7 +243,8 @@ function complete(job: Job) {
})();
}
export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null): Promise<void> {
export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null,
options: { wait?: boolean } = {}): Promise<void> {
// Draining prevents admission immediately and retires existing lobbies only through the normal safety queue.
for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) {
const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
@@ -253,12 +259,23 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at
FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
for (const job of jobs) {
try { await processJob(job, eligibility); }
catch (error) { hold(job, (error as Error).message); }
}
// One host's observation must not prevent another job being admitted.
// Per-job ownership also covers overlapping direct calls and worker passes.
const work = jobs.map(job => {
const pending = activeJobs.get(job.id);
if (pending) return pending;
const work = trackActivity(`provisioning:${job.id}`, `Evaluating ${job.action} for ${job.server}`,
() => processJob(job, eligibility))
.catch(error => { if (stillActive(job)) hold(job, (error as Error).message); })
.finally(() => { activeJobs.delete(job.id); });
activeJobs.set(job.id, work);
return work;
});
if (options.wait !== false) await Promise.all(work);
}
const activeJobs = new Map<string, Promise<void>>();
async function processJob(job: Job, eligibility: (job: Job) => string | null) {
let instance = nativeInstance(job.server);
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
@@ -289,10 +306,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
}
if (job.action === 'destroy') {
db().transaction(() => {
deploymentStage(job.id, REMOVING_SERVER, { checkpoint: { key: 'delete-deployment', label: 'Agent confirmed instance and container removal' } });
deploymentStage(job.id, REMOVING_SERVER, { phase: 'cleanup', checkpoint: { key: 'container-action', label: 'Agent confirmed instance and container removal', at: operation.updatedAt } });
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance!.id);
releaseHostPorts(instance!.id);
db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
recordProgress(job.id, { checkpoint: { key: 'cleanup', label: 'Instance retired and port reservations released' } });
complete(job);
})();
return;
@@ -304,28 +322,56 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
if (job.state !== 'awaiting-health') {
db().prepare("UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?").run(job.id);
recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` }, phase: 'verify-container' });
recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` },
checkpoint: { key: 'container-action', label: `Agent confirmed ${job.action}`, at: operation.updatedAt } });
}
const checks: CheckInput[] = [];
const check = (value: CheckInput, phase: import('../../provisioning-detail.js').ProvisioningPhase) => {
checks.push({ ...value, checkedAt: new Date().toISOString() });
recordProgress(job.id, { checks, ...(value.result === 'pass' ? { checkpoint: { key: phase, label: value.label } } : {}) });
};
deploymentStage(job.id, 'Checking container state with host agent', { phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', blocker: null, checks });
const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect');
if (!stillActive(job)) return;
if (job.action === 'stop') {
if (!['stopped','exited','created','dead'].includes(health.state)) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; }
const stopped = ['stopped','exited','created','dead'].includes(health.state);
check({ key: 'stopped', label: 'Stopped container verified', source: 'agent', result: stopped ? 'pass' : 'waiting', actual: health.state }, 'verify-stopped');
if (!stopped) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; }
db().prepare("UPDATE servers SET container_state=?,last_players=NULL,last_roster=NULL WHERE name=?").run(health.state, job.server);
complete(job); return;
}
check({ key: 'container', label: 'Container readiness verified', source: 'agent', result: health.healthy ? 'pass' : 'waiting', actual: health.state }, 'verify-container');
if (!health.healthy) { hold(job, `Waiting for container readiness: ${health.state}`); return; }
deploymentStage(job.id, 'Checking game status', { phase: 'verify-game', blocker: null });
const status = await probeServerStatus(job.server, operation.updatedAt);
if (!stillActive(job)) return;
check({ key: 'game', label: 'Fresh game status verified', source: status.ok ? status.source : null,
result: status.ok ? 'pass' : 'unknown', actual: status.ok ? status.players : null,
message: status.ok ? null : status.reason }, 'verify-game');
if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; }
if (job.action === 'create' || job.action === 'rebuild') {
deploymentStage(job.id, 'Checking selected engine map', { phase: 'verify-map', blocker: null });
const recipe = recipeForDeployment(job.id)!.recipe;
check({ key: 'map', label: 'Selected engine map verified', source: status.source,
result: status.map === recipe.expectedMap ? 'pass' : 'waiting', expected: recipe.expectedMap, actual: status.map }, 'verify-map');
if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
deploymentStage(job.id, 'Checking production Plutonium release', { phase: 'verify-plutonium', blocker: null });
await checkRelease();
const revision = await probeVersion(job.server, { operationId: job.id }); const target = freshRelease();
if (!stillActive(job)) return;
const target = freshRelease();
checks.push({ key: 'plutonium-release', label: 'Production Plutonium release', source: 'manifest',
result: target === null ? 'unknown' : 'pass', actual: target });
recordProgress(job.id, { checks });
deploymentStage(job.id, 'Checking running Plutonium revision', { phase: 'verify-plutonium', blocker: null });
const revision = await probeVersion(job.server, { operationId: job.id });
if (!stillActive(job)) return;
check({ key: 'plutonium-version', label: 'Plutonium revision verified', source: 'rcon', expected: target, actual: revision,
result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass' }, 'verify-plutonium');
if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
}
db().prepare("UPDATE servers SET container_state='running',container_checked_at=? WHERE name=?").run(new Date().toISOString(), job.server);
recordProgress(job.id, { blocker: null, checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', actual: health.state },
{ key: 'game', label: 'Game status', source: status.source, result: 'pass', actual: status.players }] });
db().prepare('UPDATE servers SET container_state=?,container_checked_at=? WHERE name=?').run(health.state, new Date().toISOString(), job.server);
recordProgress(job.id, { blocker: null, checks });
complete(job); return;
}
const reason = eligibility(job);
@@ -350,16 +396,23 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const settings = fleetSettings();
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue',
blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
}
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 });
if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
if (job.action !== 'create' && job.action !== 'start') {
// Fast controller check keeps occupied lobbies out of the operation slots.
// The agent repeats this immediately before its local mutation.
deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
if (!stillActive(job)) return;
if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
const occupancy = await probePlayerEvidence(job.server);
if (!stillActive(job)) return;
const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s
JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
@@ -379,12 +432,20 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
}
// Async occupancy checks can overlap another job claiming the final slot.
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
}
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
}
recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
let payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
if (job.action === 'create' || job.action === 'rebuild') {
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
const recipe: NativeRecipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image };
payload = createPayload(instance!, recipe);
@@ -394,13 +455,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
}
const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload,
{ instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id });
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
deploymentStage(job.id, `Host agent ${job.action} queued`, { phase: 'container-action', blocker: null,
deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null,
command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
})();
}