Remove Komodo backend in favor of native hosts
This commit is contained in:
@@ -121,7 +121,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
|
||||
SELECT collection_id, deployment_id FROM server_collection_members
|
||||
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
|
||||
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
|
||||
return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null,
|
||||
return { deployment, backend: 'native' as const, agentOperation: agentOperation ?? null,
|
||||
canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
|
||||
workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
|
||||
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
|
||||
@@ -133,7 +133,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
|
||||
})();
|
||||
}
|
||||
|
||||
// This stage is written only after Komodo has confirmed deployment deletion.
|
||||
// This stage is written only after the host agent has confirmed instance removal.
|
||||
export const REMOVING_SERVER = 'Removing server files and registry';
|
||||
|
||||
export function canCancelRemoval(id: string): boolean {
|
||||
@@ -154,15 +154,6 @@ export function cancelRemoval(id: string): boolean {
|
||||
})();
|
||||
}
|
||||
|
||||
export function claimRemoval(id: string): boolean {
|
||||
return db().prepare(`UPDATE server_deployments SET state = 'running'
|
||||
WHERE id = ? AND action = 'destroy' AND state = 'waiting'
|
||||
AND (server_deleted_at IS NULL OR (stage = ?
|
||||
AND NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = server_deployments.server)
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = server_deployments.server
|
||||
AND newer.rowid > server_deployments.rowid)))`).run(id, REMOVING_SERVER).changes > 0;
|
||||
}
|
||||
|
||||
export function canResumeCleanup(id: string): boolean {
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
||||
WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server))
|
||||
@@ -177,7 +168,7 @@ export function resumeCleanup(id: string): boolean {
|
||||
preserveOutcome(id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
|
||||
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed Komodo deletion' } });
|
||||
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed instance removal' } });
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
@@ -207,7 +198,7 @@ export function resumeRebuild(id: string): boolean {
|
||||
preserveOutcome(id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
|
||||
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed Komodo deletion' } });
|
||||
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed instance removal' } });
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
@@ -218,7 +209,7 @@ export function resumeVerification(id: string): boolean {
|
||||
if (!canResumeVerification(id)) return false;
|
||||
preserveOutcome(id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
|
||||
deploymentStage(id, 'Waiting for host agent health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
|
||||
event: { kind: 'resumed', message: 'Resumed verification of the existing server' } });
|
||||
return true;
|
||||
})();
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Execution is driven exclusively by the provisioning worker.
|
||||
export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
|
||||
export type { LifecycleOperation, RestartSource } from './requests.js';
|
||||
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextAvailablePort, nextLobbyNumber } from './instances.js';
|
||||
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextLobbyNumber } from './instances.js';
|
||||
export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';
|
||||
export { deploymentActivity } from './activity.js';
|
||||
export { restartSettling, restartRecoveryPending } from './restarts.js';
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -45,7 +45,7 @@ export function nativeLaunchSelection(input: Record<string, unknown>, game: stri
|
||||
if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== libraryId) {
|
||||
throw new Error(`The current ${game.toUpperCase()} game files changed. Refresh the launch preview.`);
|
||||
}
|
||||
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
|
||||
const image = input.image === undefined || input.image === '' ? config.defaultRuntimeImage : input.image;
|
||||
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
|
||||
const automatic = input.automatic === true || !input.hostId;
|
||||
if (!automatic) {
|
||||
@@ -673,8 +673,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
const recipe = prepared.recipe;
|
||||
db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?')
|
||||
.run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id);
|
||||
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
|
||||
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
|
||||
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
|
||||
VALUES(?,?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
|
||||
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
|
||||
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
|
||||
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
|
||||
|
||||
@@ -37,7 +37,7 @@ export function readProgress(row: { observation_json?: string | null }): Provisi
|
||||
|| (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null;
|
||||
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(String(c.key))
|
||||
&& typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result))
|
||||
&& (c.source === null || ['komodo', 'agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
|
||||
&& (c.source === null || ['agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
|
||||
&& typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null;
|
||||
if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string'
|
||||
&& typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null;
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
import { maintenanceAllowed } from '../maintenance/policies.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
|
||||
import { queueRebuildServer } from './instances.js';
|
||||
import { restartRequestInFlight } from './restarts.js';
|
||||
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { nativeInstance, unplacedNativeCreation } from '../backends/native.js';
|
||||
@@ -51,22 +49,6 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
|
||||
.get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id;
|
||||
if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id);
|
||||
}
|
||||
// Only not-yet-submitted campaign changes can share an activation. A later
|
||||
// file deployment must never be credited to an already running restart.
|
||||
// Restorations need their own activation. An older campaign can have
|
||||
// stale file preconditions after a beta; its failure must not strand
|
||||
// the rollback that would restore the server.
|
||||
const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id);
|
||||
if (source.kind === 'campaign' && !row.restarted_at && !rollback && !nativeInstance(server)) {
|
||||
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart'
|
||||
AND d.state = 'waiting' AND d.submitted_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.stage_operation_id IS NOT NULL)
|
||||
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
|
||||
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
|
||||
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id
|
||||
WHERE s.operation_id = d.id AND s.kind = 'campaign' AND c.action = 'rollback')
|
||||
ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id;
|
||||
}
|
||||
if (!id) {
|
||||
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
@@ -138,7 +120,7 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo
|
||||
|
||||
export function requestLifecycle(server: string, action: RuntimeAction) {
|
||||
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
|
||||
if (!nativeInstance(server) && !unplacedNativeCreation(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' };
|
||||
if (!nativeInstance(server) && !unplacedNativeCreation(server)) return { ok: false as const, status: 503, error: 'Server has no native host instance.' };
|
||||
return db().transaction(() => {
|
||||
const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
|
||||
if (blocker) return { ok: false as const, status: 409, error: blocker };
|
||||
@@ -187,7 +169,7 @@ function watchdogRequestGuard(server: string, incidentId: string): Extract<Lifec
|
||||
const row = getServer(server);
|
||||
if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' };
|
||||
if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' };
|
||||
if (!nativeInstance(server) && !komodoConfigured()) return { ok: false, status: 503, error: 'Configure a host before provisioning.' };
|
||||
if (!nativeInstance(server)) return { ok: false, status: 503, error: 'Configure a host before provisioning.' };
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -237,7 +219,7 @@ export function requestWatchdogLifecycle(server: string, action: 'restart' | 're
|
||||
JOIN watchdog_incidents i ON i.id = a.incident_id
|
||||
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
|
||||
AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
|
||||
if (retained && !restartRequestInFlight(server)) {
|
||||
if (retained) {
|
||||
const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
|
||||
.run(next.deploymentId, server, retained.operation_id);
|
||||
if (transferred.changes) {
|
||||
@@ -275,7 +257,7 @@ export function escalateWatchdogRestart(incidentId: string, operationId: string,
|
||||
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as
|
||||
{ operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined;
|
||||
const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at);
|
||||
if (restartRequestInFlight(operation.server) || (slot && (slot.operation_id !== operationId || slot.healthy_at))
|
||||
if ((slot && (slot.operation_id !== operationId || slot.healthy_at))
|
||||
|| !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) {
|
||||
return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' };
|
||||
}
|
||||
|
||||
@@ -1,20 +1,9 @@
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { provisioningCancelled } from './cancellation.js';
|
||||
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { probeServerStatus } from '../fleet/status-evidence.js';
|
||||
import { deploymentStage, finishDeployment } from './history.js';
|
||||
import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
|
||||
import type { RuntimeAction } from '../../provisioning.js';
|
||||
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
||||
import { recordProgress, type ProgressUpdate } from './progress.js';
|
||||
import { nativeDeployment } from '../backends/native.js';
|
||||
|
||||
// Komodo queues restart work. Don't mistake the still-running old container
|
||||
// for a completed restart when an API kick immediately follows the request.
|
||||
const HEALTH_POLL_MS = 15_000;
|
||||
const requestsInFlight = new Map<string, string>();
|
||||
export const restartRequestInFlight = (server: string) => requestsInFlight.has(server);
|
||||
|
||||
interface RestartSlot {
|
||||
server: string;
|
||||
@@ -37,7 +26,7 @@ export function restartRecoveryPending(server: string): boolean {
|
||||
/** The queued restart may still expose the previous healthy container. */
|
||||
export function restartSettling(server: string): boolean {
|
||||
const slot = slotFor(server);
|
||||
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
|
||||
return !!(slot && !slot.healthy_at
|
||||
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
|
||||
}
|
||||
|
||||
@@ -46,7 +35,6 @@ export function rolloutWaitReason(server: string, operationId = '', action: Runt
|
||||
}
|
||||
|
||||
export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
|
||||
if (requestsInFlight.has(server)) return { code: 'operation-conflict', message: 'Waiting for the previous command submission to finish', automatic: true };
|
||||
const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
|
||||
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
|
||||
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
|
||||
@@ -67,167 +55,6 @@ export function rolloutBlocker(server: string, operationId = '', action: Runtime
|
||||
? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null;
|
||||
}
|
||||
|
||||
export type RolloutRestartResult =
|
||||
| { kind: 'started' }
|
||||
| { kind: 'queued'; message: string }
|
||||
| { kind: 'failed'; message: string };
|
||||
|
||||
/** Reserve capacity before any async I/O; callers still check for players first. */
|
||||
export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
|
||||
const operationId = options.operationId;
|
||||
const waitReason = db().transaction(() => {
|
||||
if (provisioningCancelled(operationId)) return 'Operation was cancelled';
|
||||
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
|
||||
const reason = rolloutWaitReason(server, operationId, options.action);
|
||||
if (reason) return reason;
|
||||
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
||||
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
|
||||
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
|
||||
.run(server, new Date().toISOString(), operationId);
|
||||
recordProgress(operationId, { blocker: null, recovery: { state: 'held', message: 'Restart recovery capacity reserved.', operationId } });
|
||||
return null;
|
||||
}).immediate();
|
||||
if (waitReason) return { kind: 'queued', message: waitReason };
|
||||
|
||||
// Every gate, including rollout capacity, has cleared. No async work may
|
||||
// intervene between applying these files and submitting the restart.
|
||||
try {
|
||||
applyStagedFilesForOperation(operationId, server);
|
||||
} catch (err) {
|
||||
const message = `File activation failed: ${(err as Error).message}`;
|
||||
db().transaction(() => {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
|
||||
recordProgress(operationId, { recovery: { state: 'released', message: 'Capacity released before any remote command: file activation failed.', operationId } });
|
||||
finishDeployment(operationId, { ok: false, message });
|
||||
})();
|
||||
return { kind: 'failed', message };
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
|
||||
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container', {
|
||||
phase: 'container-action', blocker: null, checks: [], checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' },
|
||||
command: { state: 'attempted', message: `${options.action === 'start' ? 'Start' : 'Restart'} submission attempted.` }
|
||||
});
|
||||
})();
|
||||
requestsInFlight.set(server, operationId);
|
||||
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, rcon_failures = 0, last_reply_snippet = NULL,
|
||||
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
|
||||
try {
|
||||
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
|
||||
if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
|
||||
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
|
||||
if (result.ok) {
|
||||
db().transaction(() => {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
|
||||
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)', {
|
||||
phase: 'verify-container', command: { state: 'accepted', message: 'Komodo accepted the request; recovery still requires verification.' },
|
||||
checkpoint: { key: 'container-action', label: 'Container command accepted' },
|
||||
blocker: { code: 'recovery-settling', message: 'Waiting for the recovery settling interval before checking health.', automatic: true }
|
||||
});
|
||||
})();
|
||||
return { kind: 'started' };
|
||||
}
|
||||
// These failures cannot have queued a remote restart. Other failures
|
||||
// (especially a timeout) may have: retain capacity until recovery.
|
||||
db().transaction(() => {
|
||||
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
|
||||
recordProgress(operationId, { command: { state: 'rejected', message: 'The command could not be submitted: ' + result.message },
|
||||
recovery: { state: 'released', message: 'Capacity released because no remote command could be submitted.', operationId } });
|
||||
} else {
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
|
||||
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + result.message } });
|
||||
}
|
||||
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
|
||||
})();
|
||||
return { kind: 'failed', message: `komodo: ${result.message}` };
|
||||
} catch (err) {
|
||||
if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
|
||||
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
|
||||
const message = `komodo: ${(err as Error).message}`;
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
|
||||
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + (err as Error).message } });
|
||||
finishDeployment(operationId!, { ok: false, message });
|
||||
})();
|
||||
return { kind: 'failed', message };
|
||||
} finally {
|
||||
// Start the grace period at completion, including slow/failed requests.
|
||||
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ? AND operation_id = ?')
|
||||
.run(new Date().toISOString(), server, operationId);
|
||||
if (requestsInFlight.get(server) === operationId) requestsInFlight.delete(server);
|
||||
}
|
||||
}
|
||||
|
||||
let refreshInFlight: Promise<void> | null = null;
|
||||
|
||||
/** Poll all occupied slots, even if their originating beta was already closed. */
|
||||
export function refreshRestartRollout(): Promise<void> {
|
||||
if (refreshInFlight) return refreshInFlight;
|
||||
refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
|
||||
return refreshInFlight;
|
||||
}
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
|
||||
.all() as RestartSlot[];
|
||||
for (const slot of slots) {
|
||||
if (slot.operation_id && nativeDeployment(slot.operation_id)) continue;
|
||||
if (requestsInFlight.has(slot.server)) continue;
|
||||
// Recovery capacity transferred to a rebuild remains reserved until
|
||||
// teardown. Its health is verified by the creation worker, never here.
|
||||
if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND (
|
||||
(d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health'))
|
||||
OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (
|
||||
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`)
|
||||
.get(slot.operation_id)) continue;
|
||||
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
|
||||
let error: string | null = null;
|
||||
let players: number | null = null;
|
||||
const checks: NonNullable<ProgressUpdate['checks']> = [];
|
||||
try {
|
||||
const health = await komodoInspectHealthy(slot.server);
|
||||
checks.push({ key: 'container', label: 'Container readiness', result: health.ok ? 'pass' : 'waiting', source: 'komodo', actual: health.state ?? null,
|
||||
checkedAt: new Date().toISOString(), message: health.ok ? null : health.reason });
|
||||
if (!health.ok) {
|
||||
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
|
||||
} else {
|
||||
const status = await probeServerStatus(slot.server, slot.requested_at);
|
||||
checks.push({ key: 'game', label: 'Game status', result: status.ok ? 'pass' : 'waiting', source: status.ok ? status.source : null,
|
||||
checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null,
|
||||
message: status.ok ? null : status.reason });
|
||||
if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`;
|
||||
else players = status.players;
|
||||
}
|
||||
} catch (err) {
|
||||
error = `health check: ${(err as Error).message}`;
|
||||
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', result: 'unknown', source: null, message: error });
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
db().transaction(() => {
|
||||
const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
|
||||
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
|
||||
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
|
||||
if (!checked.changes) return;
|
||||
if (slot.operation_id) recordProgress(slot.operation_id, {
|
||||
phase: checks.some(check => check.key === 'game') ? 'verify-game' : 'verify-container', checks, observedAt: now,
|
||||
blocker: error ? { code: 'recovery-health', message: error, automatic: true } : null,
|
||||
recovery: { state: error ? 'held' : 'released', at: now, operationId: slot.operation_id,
|
||||
message: error ? 'Restart capacity remains reserved while recovery is checked.' : 'Container and game recovery verified; restart capacity released.' },
|
||||
checkpoints: error ? [] : [{ key: 'verify-container', label: 'Container readiness verified', at: now }, { key: 'verify-game', label: 'Game status verified', at: now }]
|
||||
});
|
||||
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
|
||||
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
|
||||
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
|
||||
else {
|
||||
deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
|
||||
finishDeployment(slot.operation_id, { ok: true });
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
}
|
||||
|
||||
export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
|
||||
|
||||
@@ -1,24 +1,11 @@
|
||||
import { maintenanceAllowed } from '../maintenance/policies.js';
|
||||
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { provisioningCancelled } from './cancellation.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
|
||||
import { processCreations, processDestructions, rebuildServer } from './instances.js';
|
||||
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
|
||||
import { rolloutBlocker } from './restarts.js';
|
||||
import { cancelProvisioning } from './requests.js';
|
||||
import type { RuntimeAction } from '../../provisioning.js';
|
||||
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { nativeInstance, nativeDeployment } from '../backends/native.js';
|
||||
import { processNativeProvisioning } from './native.js';
|
||||
import { trackActivity } from '../pipeline/activity.js';
|
||||
|
||||
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
|
||||
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
|
||||
|
||||
interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string }
|
||||
const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[];
|
||||
@@ -92,20 +79,13 @@ function blocker(server: string, id: string, action: RuntimeAction): Provisionin
|
||||
}
|
||||
|
||||
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
|
||||
let ticking: Promise<void> | null = null;
|
||||
let awaitedPass: Promise<void> | null = null;
|
||||
export function processProvisioning(options: { wait?: boolean } = {}): Promise<void> {
|
||||
if (options.wait !== false && awaitedPass) return awaitedPass;
|
||||
// Native jobs own their individual flights. A slow observation must not
|
||||
// prevent the next scheduler pass admitting newly arrived native work.
|
||||
const native = nativePass(options);
|
||||
if (!ticking) ticking = trackActivity('legacy-provisioning', 'Processing legacy provisioning', run)
|
||||
.finally(() => { ticking = null; });
|
||||
if (options.wait === false) {
|
||||
void ticking.catch(error => console.error('[worker] legacy provisioning error:', error));
|
||||
return native;
|
||||
}
|
||||
awaitedPass = Promise.all([native, ticking]).then(() => {}).finally(() => { awaitedPass = null; });
|
||||
// Each job owns its flight, so slow observations do not block new work.
|
||||
const pass = nativePass(options);
|
||||
if (options.wait === false) return pass;
|
||||
awaitedPass = pass.finally(() => { awaitedPass = null; });
|
||||
return awaitedPass;
|
||||
}
|
||||
|
||||
@@ -126,148 +106,3 @@ function nativePass(options: { wait?: boolean }): Promise<void> {
|
||||
return blocker(operation.server, operation.id, typed.action)?.message ?? null;
|
||||
}, options);
|
||||
}
|
||||
|
||||
async function run(): Promise<void> {
|
||||
await processCreations();
|
||||
await processDestructions();
|
||||
await refreshRestartRollout();
|
||||
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
|
||||
ORDER BY started_at, rowid`).all() as Operation[];
|
||||
for (const operation of operations) {
|
||||
if (nativeDeployment(operation.id)) continue;
|
||||
try { await processOperation(operation); }
|
||||
catch (err) {
|
||||
if (provisioningCancelled(operation.id)) continue;
|
||||
// Observation failures are retried; remote mutations are never replayed.
|
||||
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, {
|
||||
checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true }
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function processOperation(operation: Operation): Promise<void> {
|
||||
const { id, server, action } = operation;
|
||||
if (provisioningCancelled(id)) return;
|
||||
// An explicit cleanup resume follows confirmed remote deletion. Complete
|
||||
// that durable checkpoint even if the incident now needs attention or the
|
||||
// Watchdog is paused; never strand the reserved replacement identity.
|
||||
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
|
||||
if (operation.state === 'waiting') {
|
||||
const heldBy: Source[] = [];
|
||||
const eligibility = sourceEligibility(operation, source => heldBy.push(source));
|
||||
if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
|
||||
if (eligibility === 'held') {
|
||||
const source = heldBy[0];
|
||||
const href = source ? source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}`
|
||||
: source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}`
|
||||
: source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}`
|
||||
: `/maintenance?view=activity#job-${encodeURIComponent(source.source_id)}` : null;
|
||||
deploymentStage(id, 'Waiting for the requester to resume', { phase: 'eligibility', checks: [],
|
||||
blocker: { code: 'requester-held', message: 'Waiting for the requester to resume', automatic: false, href, label: 'Requesting work' } });
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
|
||||
const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog');
|
||||
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration', {
|
||||
phase: 'eligibility', checks: [], blocker: { code: 'configuration', message: 'Waiting for Komodo and RCON configuration', automatic: false, href: '/settings', label: 'Integration settings' }
|
||||
}); return; }
|
||||
if (operation.state === 'awaiting-health') {
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(id)) return;
|
||||
recordProgress(id, { phase: 'verify-stopped', checks: [{ key: 'stopped', label: 'Container stopped', source: 'komodo',
|
||||
result: isStopped(health.state) ? 'pass' : health.state ? 'waiting' : 'unknown', actual: health.state ?? null,
|
||||
message: health.ok ? null : health.reason }], blocker: isStopped(health.state) ? null
|
||||
: { code: 'stopped-state', message: 'Waiting for an explicit stopped container state.', automatic: true } });
|
||||
if (isStopped(health.state)) {
|
||||
db().transaction(() => {
|
||||
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
|
||||
deploymentStage(id, 'Verified container stopped', { checkpoint: { key: 'verify-stopped', label: 'Stopped container state verified' } });
|
||||
finishDeployment(id, { ok: true });
|
||||
})();
|
||||
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
|
||||
return;
|
||||
}
|
||||
const reason = blocker(server, id, action);
|
||||
if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
|
||||
const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s
|
||||
JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id
|
||||
WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
|
||||
const epoch = latestGameSubmission(server);
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (provisioningCancelled(id)) return;
|
||||
if (healthRecovery && epoch !== latestGameSubmission(server)) return;
|
||||
if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
|
||||
deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
|
||||
blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } });
|
||||
return;
|
||||
}
|
||||
if (action !== 'start' && !isStopped(health.state)) {
|
||||
const evidence = await probePlayerEvidence(server);
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
const players = evidence.players;
|
||||
if (healthRecovery && epoch !== latestGameSubmission(server)) return;
|
||||
if (healthRecovery && health.ok && players !== null) {
|
||||
// The fresh occupancy check also proves that the game recovered.
|
||||
// Empty is not a reason to execute an obsolete health restart.
|
||||
resolveRecoveredHealthOperation(id, players, evidence.source);
|
||||
return;
|
||||
}
|
||||
recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
|
||||
source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
|
||||
blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',
|
||||
message: players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`, automatic: true } });
|
||||
db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
|
||||
if (players !== 0) {
|
||||
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
|
||||
return;
|
||||
}
|
||||
} else if (action !== 'start') {
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
recordProgress(id, { phase: 'eligibility', blocker: null, checks: [{ key: 'stopped', label: 'Stopped container permits disruption',
|
||||
source: 'komodo', result: 'pass', actual: health.state ?? null }] });
|
||||
}
|
||||
// Requests can arrive during the probes. Recheck before claiming the action.
|
||||
const latestReason = blocker(server, id, action);
|
||||
if (latestReason) { deploymentStage(id, latestReason.message, { phase: 'eligibility', blocker: latestReason }); return; }
|
||||
if (sourceEligibility(operation) !== 'ready') return;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
if (action === 'rebuild') { await rebuildServer(id); return; }
|
||||
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id);
|
||||
if (!claim.changes) return;
|
||||
if (action === 'start' || action === 'restart') {
|
||||
const result = await restartInRollout(server, { operationId: id, action });
|
||||
if (provisioningCancelled(id)) return;
|
||||
if (result.kind === 'queued') {
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, result.message);
|
||||
}
|
||||
return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
|
||||
deploymentStage(id, 'Stopping container', { phase: 'container-action', blocker: null, checks: [],
|
||||
checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, command: { state: 'attempted', message: 'Stop submission attempted.' } });
|
||||
})();
|
||||
// Persist intent first; a lost reply must not issue another stop later.
|
||||
try {
|
||||
const result = await komodoRuntimeAction(server, 'stop');
|
||||
if (provisioningCancelled(id)) return;
|
||||
if (!result.ok) {
|
||||
recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } });
|
||||
finishDeployment(id, { ok: false, message: result.message }); return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for container to stop', { phase: 'verify-stopped',
|
||||
checkpoint: { key: 'container-action', label: 'Stop command accepted' },
|
||||
command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } });
|
||||
})();
|
||||
} catch (err) {
|
||||
if (provisioningCancelled(id)) return;
|
||||
recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } });
|
||||
finishDeployment(id, { ok: false, message: (err as Error).message });
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user