josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Cancel active provisioning jobs when rolling back betas

Browse Source
This commit is contained in:
josh
2026-10-05 17:48:46 -04:00
parent 6ad5e7f4c6
commit 701066aaf8
25 changed files with 564 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/instances.ts
+24 -6
View File
@@ -1,6 +1,7 @@
import { listHosts } from '../hosts/index.js';
import { fleetSettings } from '../settings.js';
import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js';
import { assertProvisioningNotCancelled, provisioningCancelled } from './cancellation.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js';
@@ -76,7 +77,7 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = []
const rows = db()
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.state = 'cancelled' AND d.submitted_at IS NOT NULL) OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
.all(`${baseName}_%`) as Array<{ name: string }>;
const taken = new Set<number>();
const prefix = `${baseName}_`;
@@ -100,7 +101,7 @@ export function nextAvailablePort(additionalPorts: number[] = [], releasingServe
const rows = db()
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.state = 'cancelled' AND d.submitted_at IS NOT NULL) OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
.all() as Array<{ name: string; game_port: number }>;
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
@@ -159,9 +160,14 @@ function buildSpec(args: {
}
let creating = false;
async function observeCommand<T>(operation: string, command: () => Promise<T>): Promise<T> {
try { return await command(); }
catch (err) {
async function observeCommand<T>(operation: string, command: () => Promise<T>, retainReceipt = false): Promise<T> {
assertProvisioningNotCancelled(operation);
try {
const result = await command();
if (!retainReceipt) assertProvisioningNotCancelled(operation);
return result;
} catch (err) {
if (provisioningCancelled(operation)) throw err;
recordProgress(operation, { command: { state: 'uncertain', message: 'No confirmed command result was received: ' + (err as Error).message } });
throw err;
}
@@ -314,6 +320,7 @@ export async function rebuildServer(id: string): Promise<void> {
try {
const result = job.stage === REMOVING_SERVER
? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
if (provisioningCancelled(id)) return;
if (!result.ok || result.warnings.length) {
finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
return;
@@ -435,12 +442,16 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
return { ok: false, kind: 'fs', message: `failed to create ${hostDir}: ${(err as Error).message}` };
}
assertProvisioningNotCancelled(operation);
// Step 2: Komodo CreateDeployment.
deploymentStage(operation, 'Creating Komodo deployment', { phase: 'create-deployment',
checkpoint: { key: 'prepare', label: 'Server files prepared' },
command: { state: 'attempted', message: 'Komodo deployment creation attempted.' } });
const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
const createRes = await observeCommand(operation, () => komodoCreateDeployment(spec));
db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), operation);
// A confirmed creation must retain its identity for inspection/removal even if cancelled in flight.
const createRes = await observeCommand(operation, () => komodoCreateDeployment(spec), true);
if (!createRes.ok && provisioningCancelled(operation)) assertProvisioningNotCancelled(operation);
if (!createRes.ok) {
recordProgress(operation, { command: { state: createRes.kind === 'unreachable' || (createRes.httpStatus !== undefined && createRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: createRes.message } });
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
@@ -482,6 +493,7 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
return { ok: false, kind: 'db', message: `failed to insert server row: ${(err as Error).message}` };
}
assertProvisioningNotCancelled(operation);
// Step 4: apply Komodo tags so operators can filter the deployment list
// by game/map. Best-effort — a tag failure doesn't roll back the server;
// the deployment is fully functional without cosmetic tags. Fires in
@@ -718,6 +730,7 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
const pollDeadline = Date.now() + 30_000;
while (Date.now() < pollDeadline) {
const health = await komodoInspectHealthy(name);
assertProvisioningNotCancelled(operation);
if (!health.ok) break;
await new Promise((resolve) => setTimeout(resolve, 1500));
}
@@ -740,6 +753,7 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
}
async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
assertProvisioningNotCancelled(operation);
const blocker = activeBetaRemovalBlocker(name);
if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };
const row = getServer(name);
@@ -753,11 +767,13 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
const hostDir = row?.host_dir ?? serverDir(name);
try {
await rm(hostDir, { recursive: true, force: true });
assertProvisioningNotCancelled(operation);
recordProgress(operation, { checkpoint: { key: 'files-removed', label: 'Instance files removed', phase: 'cleanup' } });
} catch (err) {
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
}
assertProvisioningNotCancelled(operation);
// The registry trigger preserves history and marks its old server identity.
// Live inventory, rollback snapshots, and restart slots still cascade away.
try {
@@ -802,6 +818,7 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
}
const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
{ ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
if (provisioningCancelled(id) && result.ok) return result;
if (!result.ok) finishDeployment(id, result);
else {
db().transaction(() => {
@@ -812,6 +829,7 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
}
return result;
} catch (err) {
if (provisioningCancelled(id)) throw err;
const server = getServer(job.server);
if (server?.game_port === job.port) {
// A lost response may have happened after Komodo started the server.