josh
e5c0fda1ed
Introduce an outbound Python host agent, Debian installer, durable remote operations, immutable game-file libraries, host-aware deployments and ports, and occupancy-safe campaigns and draining. Native hosts do not require Komodo. Add administrator, operator and viewer accounts, protected management routes, secure sessions, password recovery, audit history and separate telemetry tokens. Document first-admin setup, reporter migration and live-host acceptance checks. Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type checking, the production build, compiled SSR/API smoke tests and administrator recovery. Real Debian/Docker T5/T6 acceptance remains unrun.
256 lines
17 KiB
TypeScript
256 lines
17 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { db } from '../db/index.js';
|
|
import { recipeForDeployment, summarizeRecipe } from '../fleet/launch-recipes.js';
|
|
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
|
|
import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js';
|
|
import { fleetSettings } from '../settings.js';
|
|
import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
|
|
import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
|
|
import { bindNativeOperation } from './native.js';
|
|
|
|
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
|
|
const id = randomUUID();
|
|
db().transaction(() => {
|
|
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
|
|
bindNativeOperation(id, server);
|
|
recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested',
|
|
label: action === 'create' ? 'Server creation requested'
|
|
: action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested` } });
|
|
})();
|
|
return id;
|
|
}
|
|
|
|
export function deploymentStage(id: string, stage: string, update?: ProgressUpdate): void {
|
|
db().transaction(() => {
|
|
const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
|
|
const changed = !!previous && previous.stage !== stage;
|
|
const before = update && changed ? activitySummary(id).latestId ?? 0 : 0;
|
|
db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
|
|
if (update) recordProgress(id, update);
|
|
// Structured transitions may already carry the exact legacy stage text.
|
|
const alreadyRecorded = update && changed && db().prepare(`SELECT 1 FROM server_deployment_events
|
|
WHERE deployment_id = ? AND id > ? AND message = ? LIMIT 1`).get(id, before, stage);
|
|
if (changed && !alreadyRecorded) recordDeploymentEvent(id, stage);
|
|
})();
|
|
}
|
|
|
|
export function recordDeploymentEvent(id: string, message: string): void {
|
|
db().prepare('INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)')
|
|
.run(id, new Date().toISOString(), message);
|
|
}
|
|
|
|
function warningMessages(raw: string): string[] {
|
|
try {
|
|
const parsed: unknown = JSON.parse(raw);
|
|
if (!Array.isArray(parsed)) return ['Recorded warnings could not be read.'];
|
|
const messages = parsed.filter((value): value is string => typeof value === 'string');
|
|
if (messages.length !== parsed.length) messages.push('Some recorded warnings could not be read.');
|
|
return messages;
|
|
} catch { return ['Recorded warnings could not be read.']; }
|
|
}
|
|
|
|
function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
|
|
const id = source.source_id;
|
|
if (source.kind === 'campaign') {
|
|
const row = db().prepare('SELECT action FROM deployments WHERE id = ?').get(id) as { action: string } | undefined;
|
|
const action = row?.action === 'promote' ? 'Promotion' : row?.action === 'rollback' ? 'Rollback' : row?.action === 'drift' ? 'Reconciliation' : 'Campaign';
|
|
return { ...source, label: `${action} ${id.slice(0, 8)}`, href: row ? `/campaigns/${encodeURIComponent(id)}` : null };
|
|
}
|
|
if (source.kind === 'beta') {
|
|
const row = db().prepare('SELECT notes FROM betas WHERE id = ?').get(id) as { notes: string | null } | undefined;
|
|
const label = row?.notes?.trim().split('\n')[0].slice(0, 120);
|
|
return { ...source, label: label ? `Beta #${id}: ${label}` : `Beta #${id}`, href: row ? `/betas/${encodeURIComponent(id)}` : null };
|
|
}
|
|
if (source.kind === 'watchdog') {
|
|
const row = db().prepare('SELECT summary FROM watchdog_incidents WHERE id = ?').get(id) as { summary: string } | undefined;
|
|
return { ...source, label: row?.summary ?? `Maintenance finding ${id.slice(0, 8)}`, href: row ? `/maintenance/${encodeURIComponent(id)}` : null };
|
|
}
|
|
const row = db().prepare('SELECT kind FROM maintenance_jobs WHERE id = ?').get(id) as { kind: string } | undefined;
|
|
return { ...source, label: row && row.kind !== 'legacy' ? `Maintenance ${row.kind} #${id}` : `Maintenance #${id}`,
|
|
href: row ? `/maintenance?view=activity#job-${encodeURIComponent(id)}` : null };
|
|
}
|
|
|
|
/** These are current pool snapshots, not a second admission/eligibility engine. */
|
|
function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): ProvisioningCapacity {
|
|
const settings = fleetSettings();
|
|
const pools: ProvisioningCapacity['pools'] = [];
|
|
if (deployment.action === 'create' || deployment.action === 'rebuild') {
|
|
const jobs = db().prepare(`SELECT id, server, state FROM server_deployments
|
|
WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health')
|
|
ORDER BY started_at, rowid`).all() as { id: string; server: string; state: string }[];
|
|
const holders = jobs.filter(job => job.state !== 'queued').map(({ id, server }) => ({ id, server }));
|
|
const order = jobs.filter(job => job.state === 'queued').findIndex(job => job.id === deployment.id);
|
|
pools.push({ kind: 'startup', limit: settings.createConcurrency, used: holders.length, holders,
|
|
requestOrder: order < 0 ? null : order + 1, ownsSlot: holders.some(holder => holder.id === deployment.id) });
|
|
}
|
|
if (['start', 'restart', 'rebuild'].includes(deployment.action)) {
|
|
const holders = db().prepare(`SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL
|
|
ORDER BY requested_at, server`).all() as { id: string | null; server: string }[];
|
|
pools.push({ kind: 'restart', limit: settings.restartConcurrency, used: holders.length, holders,
|
|
requestOrder: null, ownsSlot: holders.some(holder => holder.id === deployment.id) });
|
|
}
|
|
return { evaluatedAt, pools };
|
|
}
|
|
|
|
/** The default retains the full legacy event response. Page clients opt into a bounded window. */
|
|
export function getDeployment(id: string, options: { events?: 'all' | 'recent' } = {}): ProvisioningDetail | null {
|
|
return db().transaction(() => {
|
|
const row = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
|
|
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id) as
|
|
(ProvisioningDeployment & { observation_json: string | null; phase_key: string | null; phase_started_at: string | null; observed_at: string | null }) | undefined;
|
|
if (!row) return null;
|
|
const generatedAt = new Date().toISOString();
|
|
const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
|
|
const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
|
|
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
|
|
const launch = recipeForDeployment(id);
|
|
const collection = db().prepare(`SELECT c.id, c.name FROM (
|
|
SELECT collection_id, deployment_id FROM server_collection_members
|
|
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
|
|
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
|
|
return { deployment, events: deploymentEvents(id, options.events === 'recent'),
|
|
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
|
|
recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
|
|
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
|
|
canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
|
|
progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
|
|
warningMessages: warningMessages(deployment.warnings), generatedAt };
|
|
})();
|
|
}
|
|
|
|
// This stage is written only after Komodo has confirmed deployment deletion.
|
|
export const REMOVING_SERVER = 'Removing server files and registry';
|
|
|
|
export function canCancelRemoval(id: string): boolean {
|
|
return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
|
|
}
|
|
|
|
/** Competes atomically with the worker's claim, before any external removal. */
|
|
export function cancelRemoval(id: string): boolean {
|
|
return db().transaction(() => {
|
|
const finishedAt = new Date().toISOString();
|
|
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
|
|
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(finishedAt, id, REMOVING_SERVER);
|
|
if (!result.changes) return false;
|
|
recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome',
|
|
message: 'Queued removal cancelled. The server and its files were kept.',
|
|
details: { state: 'cancelled', finishedAt } } });
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
export function claimRemoval(id: string): boolean {
|
|
return db().prepare(`UPDATE server_deployments SET state = 'running'
|
|
WHERE id = ? AND action = 'destroy' AND state = 'waiting'
|
|
AND (server_deleted_at IS NULL OR (stage = ?
|
|
AND NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = server_deployments.server)
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = server_deployments.server
|
|
AND newer.rowid > server_deployments.rowid)))`).run(id, REMOVING_SERVER).changes > 0;
|
|
}
|
|
|
|
export function canResumeCleanup(id: string): boolean {
|
|
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
|
WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server))
|
|
AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
|
.get(id, REMOVING_SERVER);
|
|
}
|
|
|
|
export function resumeCleanup(id: string): boolean {
|
|
return db().transaction(() => {
|
|
if (!canResumeCleanup(id)) return false;
|
|
preserveOutcome(id);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
|
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
|
|
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed Komodo deletion' } });
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
export function canResumeVerification(id: string): boolean {
|
|
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
|
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
|
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
|
|
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
|
.get(id);
|
|
}
|
|
|
|
export function canResumeRebuild(id: string): boolean {
|
|
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
|
.get(id, REMOVING_SERVER);
|
|
}
|
|
|
|
export function resumeRebuild(id: string): boolean {
|
|
return db().transaction(() => {
|
|
if (!canResumeRebuild(id)) return false;
|
|
preserveOutcome(id);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
|
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
|
|
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed Komodo deletion' } });
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
/** Resume observation only. Never repeat a container creation or restart. */
|
|
export function resumeVerification(id: string): boolean {
|
|
return db().transaction(() => {
|
|
if (!canResumeVerification(id)) return false;
|
|
preserveOutcome(id);
|
|
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
|
|
event: { kind: 'resumed', message: 'Resumed verification of the existing server' } });
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
|
|
const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
|
|
db().transaction(() => {
|
|
const finishedAt = new Date().toISOString();
|
|
db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
|
|
.run(state, finishedAt, result.message ?? null, JSON.stringify(result.warnings ?? []), id);
|
|
for (const warning of result.warnings ?? []) recordProgress(id, { event: { kind: 'warning', message: warning } });
|
|
recordProgress(id, { blocker: null, event: { kind: 'outcome',
|
|
message: result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`,
|
|
details: { state, finishedAt, error: result.message ?? null, warnings: result.warnings ?? [] } } });
|
|
})();
|
|
}
|
|
|
|
/** Legacy failures also need a durable structured result before same-ID resume clears it. */
|
|
function preserveOutcome(id: string): void {
|
|
const row = db().prepare('SELECT state, finished_at, error, warnings, observation_json FROM server_deployments WHERE id = ?').get(id) as
|
|
{ state: string; finished_at: string | null; error: string | null; warnings: string; observation_json: string | null } | undefined;
|
|
if (!row || !['failed', 'interrupted', 'warning', 'done', 'cancelled'].includes(row.state)) return;
|
|
const previous = db().prepare("SELECT details_json FROM server_deployment_events WHERE deployment_id = ? AND kind = 'outcome' ORDER BY id DESC LIMIT 1")
|
|
.get(id) as { details_json: string | null } | undefined;
|
|
try {
|
|
const details = JSON.parse(previous?.details_json ?? 'null');
|
|
if (details?.state === row.state && details?.finishedAt === row.finished_at) return;
|
|
} catch { /* Preserve the row's known outcome when old event metadata is missing. */ }
|
|
recordProgress(id, { observedAt: readProgress(row)?.observedAt ?? null, event: { kind: 'outcome',
|
|
message: `${row.state === 'interrupted' ? 'Interrupted' : 'Recorded outcome'}: ${row.error ?? row.state}`,
|
|
details: { state: row.state, finishedAt: row.finished_at, error: row.error, warnings: warningMessages(row.warnings) } } });
|
|
}
|
|
|
|
export function listDeployments() {
|
|
return db().prepare(`SELECT d.*, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists,
|
|
(d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
|
|
FROM server_deployments d ORDER BY started_at DESC, rowid DESC`).all(REMOVING_SERVER);
|
|
}
|
|
|
|
export function interruptDeployments(): void {
|
|
db().transaction(() => {
|
|
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)").all() as { id: string }[];
|
|
const finishedAt = new Date().toISOString();
|
|
const error = 'The application restarted before this operation finished. Check the server before retrying.';
|
|
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)`).run(finishedAt, error);
|
|
for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome',
|
|
message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } });
|
|
})();
|
|
}
|