josh
e5c0fda1ed
Introduce an outbound Python host agent, Debian installer, durable remote operations, immutable game-file libraries, host-aware deployments and ports, and occupancy-safe campaigns and draining. Native hosts do not require Komodo. Add administrator, operator and viewer accounts, protected management routes, secure sessions, password recovery, audit history and separate telemetry tokens. Document first-admin setup, reporter migration and live-host acceptance checks. Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type checking, the production build, compiled SSR/API smoke tests and administrator recovery. Real Debian/Docker T5/T6 acceptance remains unrun.
231 lines
15 KiB
TypeScript
231 lines
15 KiB
TypeScript
import { fleetSettings } from '../settings.js';
|
|
import { db } from '../db/index.js';
|
|
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
|
|
import { probeServerStatus } from '../fleet/status-evidence.js';
|
|
import { deploymentStage, finishDeployment } from './history.js';
|
|
import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
|
|
import type { RuntimeAction } from '../../provisioning.js';
|
|
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
|
import { recordProgress, type ProgressUpdate } from './progress.js';
|
|
import { nativeDeployment } from '../backends/native.js';
|
|
|
|
// Komodo queues restart work. Don't mistake the still-running old container
|
|
// for a completed restart when an API kick immediately follows the request.
|
|
const HEALTH_POLL_MS = 15_000;
|
|
const requestsInFlight = new Map<string, string>();
|
|
export const restartRequestInFlight = (server: string) => requestsInFlight.has(server);
|
|
|
|
interface RestartSlot {
|
|
server: string;
|
|
requested_at: string;
|
|
checked_at: string | null;
|
|
healthy_at: string | null;
|
|
players: number | null;
|
|
error: string | null;
|
|
operation_id: string | null;
|
|
}
|
|
|
|
function slotFor(server: string): RestartSlot | undefined {
|
|
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
|
|
}
|
|
|
|
export function restartRecoveryPending(server: string): boolean {
|
|
return slotFor(server)?.healthy_at === null;
|
|
}
|
|
|
|
/** The queued restart may still expose the previous healthy container. */
|
|
export function restartSettling(server: string): boolean {
|
|
const slot = slotFor(server);
|
|
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
|
|
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
|
|
}
|
|
|
|
export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null {
|
|
return rolloutBlocker(server, operationId, action)?.message ?? null;
|
|
}
|
|
|
|
export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
|
|
const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
|
|
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
|
|
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
|
|
if (conflict) {
|
|
return { code: 'operation-conflict', message: 'Waiting for the current provisioning operation', automatic: true,
|
|
href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' };
|
|
}
|
|
const slot = slotFor(server);
|
|
const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS (
|
|
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId);
|
|
if (slot?.healthy_at === null && !ownHandoff) {
|
|
return { code: 'previous-recovery', message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`, automatic: true,
|
|
href: slot.operation_id ? `/provisioning/${encodeURIComponent(slot.operation_id)}` : null, label: 'Previous operation' };
|
|
}
|
|
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
|
|
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency
|
|
? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null;
|
|
}
|
|
|
|
export type RolloutRestartResult =
|
|
| { kind: 'started' }
|
|
| { kind: 'queued'; message: string }
|
|
| { kind: 'failed'; message: string };
|
|
|
|
/** Reserve capacity before any async I/O; callers still check for players first. */
|
|
export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
|
|
const operationId = options.operationId;
|
|
const waitReason = db().transaction(() => {
|
|
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
|
|
const reason = rolloutWaitReason(server, operationId, options.action);
|
|
if (reason) return reason;
|
|
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
|
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
|
|
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
|
|
.run(server, new Date().toISOString(), operationId);
|
|
recordProgress(operationId, { blocker: null, recovery: { state: 'held', message: 'Restart recovery capacity reserved.', operationId } });
|
|
return null;
|
|
}).immediate();
|
|
if (waitReason) return { kind: 'queued', message: waitReason };
|
|
|
|
// Every gate, including rollout capacity, has cleared. No async work may
|
|
// intervene between applying these files and submitting the restart.
|
|
try {
|
|
applyStagedFilesForOperation(operationId, server);
|
|
} catch (err) {
|
|
const message = `File activation failed: ${(err as Error).message}`;
|
|
db().transaction(() => {
|
|
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
|
|
recordProgress(operationId, { recovery: { state: 'released', message: 'Capacity released before any remote command: file activation failed.', operationId } });
|
|
finishDeployment(operationId, { ok: false, message });
|
|
})();
|
|
return { kind: 'failed', message };
|
|
}
|
|
db().transaction(() => {
|
|
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
|
|
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container', {
|
|
phase: 'container-action', blocker: null, checks: [], checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' },
|
|
command: { state: 'attempted', message: `${options.action === 'start' ? 'Start' : 'Restart'} submission attempted.` }
|
|
});
|
|
})();
|
|
requestsInFlight.set(server, operationId);
|
|
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
|
|
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
|
|
try {
|
|
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
|
|
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
|
|
if (result.ok) {
|
|
db().transaction(() => {
|
|
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
|
|
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)', {
|
|
phase: 'verify-container', command: { state: 'accepted', message: 'Komodo accepted the request; recovery still requires verification.' },
|
|
checkpoint: { key: 'container-action', label: 'Container command accepted' },
|
|
blocker: { code: 'recovery-settling', message: 'Waiting for the recovery settling interval before checking health.', automatic: true }
|
|
});
|
|
})();
|
|
return { kind: 'started' };
|
|
}
|
|
// These failures cannot have queued a remote restart. Other failures
|
|
// (especially a timeout) may have: retain capacity until recovery.
|
|
db().transaction(() => {
|
|
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
|
|
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
|
|
recordProgress(operationId, { command: { state: 'rejected', message: 'The command could not be submitted: ' + result.message },
|
|
recovery: { state: 'released', message: 'Capacity released because no remote command could be submitted.', operationId } });
|
|
} else {
|
|
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
|
|
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + result.message } });
|
|
}
|
|
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
|
|
})();
|
|
return { kind: 'failed', message: `komodo: ${result.message}` };
|
|
} catch (err) {
|
|
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
|
|
const message = `komodo: ${(err as Error).message}`;
|
|
db().transaction(() => {
|
|
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
|
|
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + (err as Error).message } });
|
|
finishDeployment(operationId!, { ok: false, message });
|
|
})();
|
|
return { kind: 'failed', message };
|
|
} finally {
|
|
// Start the grace period at completion, including slow/failed requests.
|
|
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ? AND operation_id = ?')
|
|
.run(new Date().toISOString(), server, operationId);
|
|
if (requestsInFlight.get(server) === operationId) requestsInFlight.delete(server);
|
|
}
|
|
}
|
|
|
|
let refreshInFlight: Promise<void> | null = null;
|
|
|
|
/** Poll all occupied slots, even if their originating beta was already closed. */
|
|
export function refreshRestartRollout(): Promise<void> {
|
|
if (refreshInFlight) return refreshInFlight;
|
|
refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
|
|
return refreshInFlight;
|
|
}
|
|
|
|
async function refresh(): Promise<void> {
|
|
const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
|
|
.all() as RestartSlot[];
|
|
for (const slot of slots) {
|
|
if (slot.operation_id && nativeDeployment(slot.operation_id)) continue;
|
|
if (requestsInFlight.has(slot.server)) continue;
|
|
// Recovery capacity transferred to a rebuild remains reserved until
|
|
// teardown. Its health is verified by the creation worker, never here.
|
|
if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND (
|
|
(d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health'))
|
|
OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (
|
|
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`)
|
|
.get(slot.operation_id)) continue;
|
|
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
|
|
let error: string | null = null;
|
|
let players: number | null = null;
|
|
const checks: NonNullable<ProgressUpdate['checks']> = [];
|
|
try {
|
|
const health = await komodoInspectHealthy(slot.server);
|
|
checks.push({ key: 'container', label: 'Container readiness', result: health.ok ? 'pass' : 'waiting', source: 'komodo', actual: health.state ?? null,
|
|
checkedAt: new Date().toISOString(), message: health.ok ? null : health.reason });
|
|
if (!health.ok) {
|
|
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
|
|
} else {
|
|
const status = await probeServerStatus(slot.server, slot.requested_at);
|
|
checks.push({ key: 'game', label: 'Game status', result: status.ok ? 'pass' : 'waiting', source: status.ok ? status.source : null,
|
|
checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null,
|
|
message: status.ok ? null : status.reason });
|
|
if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`;
|
|
else players = status.players;
|
|
}
|
|
} catch (err) {
|
|
error = `health check: ${(err as Error).message}`;
|
|
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', result: 'unknown', source: null, message: error });
|
|
}
|
|
const now = new Date().toISOString();
|
|
db().transaction(() => {
|
|
const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
|
|
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
|
|
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
|
|
if (!checked.changes) return;
|
|
if (slot.operation_id) recordProgress(slot.operation_id, {
|
|
phase: checks.some(check => check.key === 'game') ? 'verify-game' : 'verify-container', checks, observedAt: now,
|
|
blocker: error ? { code: 'recovery-health', message: error, automatic: true } : null,
|
|
recovery: { state: error ? 'held' : 'released', at: now, operationId: slot.operation_id,
|
|
message: error ? 'Restart capacity remains reserved while recovery is checked.' : 'Container and game recovery verified; restart capacity released.' },
|
|
checkpoints: error ? [] : [{ key: 'verify-container', label: 'Container readiness verified', at: now }, { key: 'verify-game', label: 'Game status verified', at: now }]
|
|
});
|
|
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
|
|
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
|
|
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
|
|
else {
|
|
deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
|
|
finishDeployment(slot.operation_id, { ok: true });
|
|
}
|
|
}
|
|
})();
|
|
}
|
|
}
|
|
|
|
export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
|
|
const slot = slotFor(server);
|
|
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
|
|
return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' };
|
|
}
|