josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Komodo backend in favor of native hosts

Browse Source
This commit is contained in:
josh
2026-10-05 18:07:07 -04:00
parent 701066aaf8
commit 4b0f341d64
116 changed files with 557 additions and 8132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/worker.ts
+5 -170
View File
@@ -1,24 +1,11 @@
import { maintenanceAllowed } from '../maintenance/policies.js';
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
import { db } from '../db/index.js';
import { provisioningCancelled } from './cancellation.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { getServer } from '../fleet/servers.js';
import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
import { processCreations, processDestructions, rebuildServer } from './instances.js';
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js';
import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
import { rolloutBlocker } from './restarts.js';
import { cancelProvisioning } from './requests.js';
import type { RuntimeAction } from '../../provisioning.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress } from './progress.js';
import { nativeInstance, nativeDeployment } from '../backends/native.js';
import { processNativeProvisioning } from './native.js';
import { trackActivity } from '../pipeline/activity.js';
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string }
const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[];
@@ -92,20 +79,13 @@ function blocker(server: string, id: string, action: RuntimeAction): Provisionin
}
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
let ticking: Promise<void> | null = null;
let awaitedPass: Promise<void> | null = null;
export function processProvisioning(options: { wait?: boolean } = {}): Promise<void> {
if (options.wait !== false && awaitedPass) return awaitedPass;
// Native jobs own their individual flights. A slow observation must not
// prevent the next scheduler pass admitting newly arrived native work.
const native = nativePass(options);
if (!ticking) ticking = trackActivity('legacy-provisioning', 'Processing legacy provisioning', run)
.finally(() => { ticking = null; });
if (options.wait === false) {
void ticking.catch(error => console.error('[worker] legacy provisioning error:', error));
return native;
}
awaitedPass = Promise.all([native, ticking]).then(() => {}).finally(() => { awaitedPass = null; });
// Each job owns its flight, so slow observations do not block new work.
const pass = nativePass(options);
if (options.wait === false) return pass;
awaitedPass = pass.finally(() => { awaitedPass = null; });
return awaitedPass;
}
@@ -126,148 +106,3 @@ function nativePass(options: { wait?: boolean }): Promise<void> {
return blocker(operation.server, operation.id, typed.action)?.message ?? null;
}, options);
}
async function run(): Promise<void> {
await processCreations();
await processDestructions();
await refreshRestartRollout();
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
ORDER BY started_at, rowid`).all() as Operation[];
for (const operation of operations) {
if (nativeDeployment(operation.id)) continue;
try { await processOperation(operation); }
catch (err) {
if (provisioningCancelled(operation.id)) continue;
// Observation failures are retried; remote mutations are never replayed.
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, {
checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true }
});
}
}
}
async function processOperation(operation: Operation): Promise<void> {
const { id, server, action } = operation;
if (provisioningCancelled(id)) return;
// An explicit cleanup resume follows confirmed remote deletion. Complete
// that durable checkpoint even if the incident now needs attention or the
// Watchdog is paused; never strand the reserved replacement identity.
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
if (operation.state === 'waiting') {
const heldBy: Source[] = [];
const eligibility = sourceEligibility(operation, source => heldBy.push(source));
if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
if (eligibility === 'held') {
const source = heldBy[0];
const href = source ? source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}`
: source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}`
: source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}`
: `/maintenance?view=activity#job-${encodeURIComponent(source.source_id)}` : null;
deploymentStage(id, 'Waiting for the requester to resume', { phase: 'eligibility', checks: [],
blocker: { code: 'requester-held', message: 'Waiting for the requester to resume', automatic: false, href, label: 'Requesting work' } });
return;
}
}
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog');
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration', {
phase: 'eligibility', checks: [], blocker: { code: 'configuration', message: 'Waiting for Komodo and RCON configuration', automatic: false, href: '/settings', label: 'Integration settings' }
}); return; }
if (operation.state === 'awaiting-health') {
const health = await komodoInspectHealthy(server);
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(id)) return;
recordProgress(id, { phase: 'verify-stopped', checks: [{ key: 'stopped', label: 'Container stopped', source: 'komodo',
result: isStopped(health.state) ? 'pass' : health.state ? 'waiting' : 'unknown', actual: health.state ?? null,
message: health.ok ? null : health.reason }], blocker: isStopped(health.state) ? null
: { code: 'stopped-state', message: 'Waiting for an explicit stopped container state.', automatic: true } });
if (isStopped(health.state)) {
db().transaction(() => {
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
deploymentStage(id, 'Verified container stopped', { checkpoint: { key: 'verify-stopped', label: 'Stopped container state verified' } });
finishDeployment(id, { ok: true });
})();
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
return;
}
const reason = blocker(server, id, action);
if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s
JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id
WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
const epoch = latestGameSubmission(server);
const health = await komodoInspectHealthy(server);
if (provisioningCancelled(id)) return;
if (healthRecovery && epoch !== latestGameSubmission(server)) return;
if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } });
return;
}
if (action !== 'start' && !isStopped(health.state)) {
const evidence = await probePlayerEvidence(server);
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
const players = evidence.players;
if (healthRecovery && epoch !== latestGameSubmission(server)) return;
if (healthRecovery && health.ok && players !== null) {
// The fresh occupancy check also proves that the game recovered.
// Empty is not a reason to execute an obsolete health restart.
resolveRecoveredHealthOperation(id, players, evidence.source);
return;
}
recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',
message: players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`, automatic: true } });
db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
if (players !== 0) {
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
return;
}
} else if (action !== 'start') {
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
recordProgress(id, { phase: 'eligibility', blocker: null, checks: [{ key: 'stopped', label: 'Stopped container permits disruption',
source: 'komodo', result: 'pass', actual: health.state ?? null }] });
}
// Requests can arrive during the probes. Recheck before claiming the action.
const latestReason = blocker(server, id, action);
if (latestReason) { deploymentStage(id, latestReason.message, { phase: 'eligibility', blocker: latestReason }); return; }
if (sourceEligibility(operation) !== 'ready') return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
if (action === 'rebuild') { await rebuildServer(id); return; }
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id);
if (!claim.changes) return;
if (action === 'start' || action === 'restart') {
const result = await restartInRollout(server, { operationId: id, action });
if (provisioningCancelled(id)) return;
if (result.kind === 'queued') {
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, result.message);
}
return;
}
db().transaction(() => {
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
deploymentStage(id, 'Stopping container', { phase: 'container-action', blocker: null, checks: [],
checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, command: { state: 'attempted', message: 'Stop submission attempted.' } });
})();
// Persist intent first; a lost reply must not issue another stop later.
try {
const result = await komodoRuntimeAction(server, 'stop');
if (provisioningCancelled(id)) return;
if (!result.ok) {
recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } });
finishDeployment(id, { ok: false, message: result.message }); return;
}
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for container to stop', { phase: 'verify-stopped',
checkpoint: { key: 'container-action', label: 'Stop command accepted' },
command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } });
})();
} catch (err) {
if (provisioningCancelled(id)) return;
recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } });
finishDeployment(id, { ok: false, message: (err as Error).message });
}
}