265 lines
18 KiB
TypeScript
265 lines
18 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { db } from '../db/index.js';
|
|
import { recipeForDeployment, summarizeRecipe } from '../fleet/launch-recipes.js';
|
|
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
|
|
import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js';
|
|
import { fleetSettings } from '../settings.js';
|
|
import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
|
|
import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
|
|
import { bindNativeOperation } from './native.js';
|
|
import { workerActivity } from '../pipeline/activity.js';
|
|
import { canCancelCreation } from './cancel-creation.js';
|
|
|
|
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
|
|
const id = randomUUID();
|
|
db().transaction(() => {
|
|
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
|
|
bindNativeOperation(id, server);
|
|
recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested',
|
|
label: action === 'create' ? 'Server creation requested'
|
|
: action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested` } });
|
|
})();
|
|
return id;
|
|
}
|
|
|
|
export function deploymentStage(id: string, stage: string, update?: ProgressUpdate): void {
|
|
db().transaction(() => {
|
|
const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
|
|
const changed = !!previous && previous.stage !== stage;
|
|
const before = update && changed ? activitySummary(id).latestId ?? 0 : 0;
|
|
db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
|
|
if (update) recordProgress(id, update);
|
|
// Structured transitions may already carry the exact legacy stage text.
|
|
const alreadyRecorded = update && changed && db().prepare(`SELECT 1 FROM server_deployment_events
|
|
WHERE deployment_id = ? AND id > ? AND message = ? LIMIT 1`).get(id, before, stage);
|
|
if (changed && !alreadyRecorded) recordDeploymentEvent(id, stage);
|
|
})();
|
|
}
|
|
|
|
export function recordDeploymentEvent(id: string, message: string): void {
|
|
db().prepare('INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)')
|
|
.run(id, new Date().toISOString(), message);
|
|
}
|
|
|
|
function warningMessages(raw: string): string[] {
|
|
try {
|
|
const parsed: unknown = JSON.parse(raw);
|
|
if (!Array.isArray(parsed)) return ['Recorded warnings could not be read.'];
|
|
const messages = parsed.filter((value): value is string => typeof value === 'string');
|
|
if (messages.length !== parsed.length) messages.push('Some recorded warnings could not be read.');
|
|
return messages;
|
|
} catch { return ['Recorded warnings could not be read.']; }
|
|
}
|
|
|
|
function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
|
|
const id = source.source_id;
|
|
if (source.kind === 'campaign') {
|
|
const row = db().prepare('SELECT action FROM deployments WHERE id = ?').get(id) as { action: string } | undefined;
|
|
const action = row?.action === 'promote' ? 'Promotion' : row?.action === 'rollback' ? 'Rollback' : row?.action === 'drift' ? 'Reconciliation' : 'Campaign';
|
|
return { ...source, label: `${action} ${id.slice(0, 8)}`, href: row ? `/campaigns/${encodeURIComponent(id)}` : null };
|
|
}
|
|
if (source.kind === 'beta') {
|
|
const row = db().prepare('SELECT notes FROM betas WHERE id = ?').get(id) as { notes: string | null } | undefined;
|
|
const label = row?.notes?.trim().split('\n')[0].slice(0, 120);
|
|
return { ...source, label: label ? `Beta #${id}: ${label}` : `Beta #${id}`, href: row ? `/betas/${encodeURIComponent(id)}` : null };
|
|
}
|
|
if (source.kind === 'watchdog') {
|
|
const row = db().prepare('SELECT summary FROM watchdog_incidents WHERE id = ?').get(id) as { summary: string } | undefined;
|
|
return { ...source, label: row?.summary ?? `Maintenance finding ${id.slice(0, 8)}`, href: row ? `/maintenance/${encodeURIComponent(id)}` : null };
|
|
}
|
|
const row = db().prepare('SELECT kind FROM maintenance_jobs WHERE id = ?').get(id) as { kind: string } | undefined;
|
|
return { ...source, label: row && row.kind !== 'legacy' ? `Maintenance ${row.kind} #${id}` : `Maintenance #${id}`,
|
|
href: row ? `/maintenance?view=activity#job-${encodeURIComponent(id)}` : null };
|
|
}
|
|
|
|
/** These are current pool snapshots, not a second admission/eligibility engine. */
|
|
function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): ProvisioningCapacity {
|
|
const settings = fleetSettings();
|
|
const pools: ProvisioningCapacity['pools'] = [];
|
|
if (deployment.action === 'create' || deployment.action === 'rebuild') {
|
|
const jobs = db().prepare(`SELECT id, server, state FROM server_deployments
|
|
WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health')
|
|
ORDER BY started_at, rowid`).all() as { id: string; server: string; state: string }[];
|
|
const holders = jobs.filter(job => job.state !== 'queued').map(({ id, server }) => ({ id, server }));
|
|
const order = jobs.filter(job => job.state === 'queued').findIndex(job => job.id === deployment.id);
|
|
pools.push({ kind: 'startup', limit: settings.createConcurrency, used: holders.length, holders,
|
|
requestOrder: order < 0 ? null : order + 1, ownsSlot: holders.some(holder => holder.id === deployment.id) });
|
|
}
|
|
if (['start', 'restart', 'rebuild'].includes(deployment.action)) {
|
|
const holders = db().prepare(`SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL
|
|
ORDER BY requested_at, server`).all() as { id: string | null; server: string }[];
|
|
pools.push({ kind: 'restart', limit: settings.restartConcurrency, used: holders.length, holders,
|
|
requestOrder: null, ownsSlot: holders.some(holder => holder.id === deployment.id) });
|
|
}
|
|
return { evaluatedAt, pools };
|
|
}
|
|
|
|
/** The default retains the full legacy event response. Page clients opt into a bounded window. */
|
|
export function getDeployment(id: string, options: { events?: 'all' | 'recent' } = {}): ProvisioningDetail | null {
|
|
return db().transaction(() => {
|
|
const row = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
|
|
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id) as
|
|
(ProvisioningDeployment & { observation_json: string | null; phase_key: string | null; phase_started_at: string | null; observed_at: string | null }) | undefined;
|
|
if (!row) return null;
|
|
const generatedAt = new Date().toISOString();
|
|
const native = db().prepare('SELECT instance_id FROM native_provisioning WHERE deployment_id=?').get(id);
|
|
const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
|
|
FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
|
|
const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
|
|
JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`).get(id) as ProvisioningDetail['cancellation'];
|
|
const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
|
|
const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
|
|
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
|
|
const launch = recipeForDeployment(id);
|
|
const collection = db().prepare(`SELECT c.id, c.name FROM (
|
|
SELECT collection_id, deployment_id FROM server_collection_members
|
|
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
|
|
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
|
|
return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null,
|
|
canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
|
|
workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
|
|
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
|
|
recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
|
|
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
|
|
canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
|
|
progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
|
|
warningMessages: warningMessages(deployment.warnings), generatedAt };
|
|
})();
|
|
}
|
|
|
|
// This stage is written only after Komodo has confirmed deployment deletion.
|
|
export const REMOVING_SERVER = 'Removing server files and registry';
|
|
|
|
export function canCancelRemoval(id: string): boolean {
|
|
return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
|
|
}
|
|
|
|
/** Competes atomically with the worker's claim, before any external removal. */
|
|
export function cancelRemoval(id: string): boolean {
|
|
return db().transaction(() => {
|
|
const finishedAt = new Date().toISOString();
|
|
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
|
|
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(finishedAt, id, REMOVING_SERVER);
|
|
if (!result.changes) return false;
|
|
recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome',
|
|
message: 'Queued removal cancelled. The server and its files were kept.',
|
|
details: { state: 'cancelled', finishedAt } } });
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
export function claimRemoval(id: string): boolean {
|
|
return db().prepare(`UPDATE server_deployments SET state = 'running'
|
|
WHERE id = ? AND action = 'destroy' AND state = 'waiting'
|
|
AND (server_deleted_at IS NULL OR (stage = ?
|
|
AND NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = server_deployments.server)
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = server_deployments.server
|
|
AND newer.rowid > server_deployments.rowid)))`).run(id, REMOVING_SERVER).changes > 0;
|
|
}
|
|
|
|
export function canResumeCleanup(id: string): boolean {
|
|
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
|
WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server))
|
|
AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
|
.get(id, REMOVING_SERVER);
|
|
}
|
|
|
|
export function resumeCleanup(id: string): boolean {
|
|
return db().transaction(() => {
|
|
if (!canResumeCleanup(id)) return false;
|
|
preserveOutcome(id);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
|
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
|
|
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed Komodo deletion' } });
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
export function canResumeVerification(id: string): boolean {
|
|
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
|
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
|
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
|
|
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
|
.get(id);
|
|
}
|
|
|
|
export function canResumeRebuild(id: string): boolean {
|
|
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
|
.get(id, REMOVING_SERVER);
|
|
}
|
|
|
|
export function resumeRebuild(id: string): boolean {
|
|
return db().transaction(() => {
|
|
if (!canResumeRebuild(id)) return false;
|
|
preserveOutcome(id);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
|
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
|
|
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed Komodo deletion' } });
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
/** Resume observation only. Never repeat a container creation or restart. */
|
|
export function resumeVerification(id: string): boolean {
|
|
return db().transaction(() => {
|
|
if (!canResumeVerification(id)) return false;
|
|
preserveOutcome(id);
|
|
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
|
|
event: { kind: 'resumed', message: 'Resumed verification of the existing server' } });
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
|
|
const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
|
|
db().transaction(() => {
|
|
const finishedAt = new Date().toISOString();
|
|
db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
|
|
.run(state, finishedAt, result.message ?? null, JSON.stringify(result.warnings ?? []), id);
|
|
for (const warning of result.warnings ?? []) recordProgress(id, { event: { kind: 'warning', message: warning } });
|
|
recordProgress(id, { blocker: null, event: { kind: 'outcome',
|
|
message: result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`,
|
|
details: { state, finishedAt, error: result.message ?? null, warnings: result.warnings ?? [] } } });
|
|
})();
|
|
}
|
|
|
|
/** Legacy failures also need a durable structured result before same-ID resume clears it. */
|
|
function preserveOutcome(id: string): void {
|
|
const row = db().prepare('SELECT state, finished_at, error, warnings, observation_json FROM server_deployments WHERE id = ?').get(id) as
|
|
{ state: string; finished_at: string | null; error: string | null; warnings: string; observation_json: string | null } | undefined;
|
|
if (!row || !['failed', 'interrupted', 'warning', 'done', 'cancelled'].includes(row.state)) return;
|
|
const previous = db().prepare("SELECT details_json FROM server_deployment_events WHERE deployment_id = ? AND kind = 'outcome' ORDER BY id DESC LIMIT 1")
|
|
.get(id) as { details_json: string | null } | undefined;
|
|
try {
|
|
const details = JSON.parse(previous?.details_json ?? 'null');
|
|
if (details?.state === row.state && details?.finishedAt === row.finished_at) return;
|
|
} catch { /* Preserve the row's known outcome when old event metadata is missing. */ }
|
|
recordProgress(id, { observedAt: readProgress(row)?.observedAt ?? null, event: { kind: 'outcome',
|
|
message: `${row.state === 'interrupted' ? 'Interrupted' : 'Recorded outcome'}: ${row.error ?? row.state}`,
|
|
details: { state: row.state, finishedAt: row.finished_at, error: row.error, warnings: warningMessages(row.warnings) } } });
|
|
}
|
|
|
|
export function listDeployments() {
|
|
return db().prepare(`SELECT d.*, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists,
|
|
(d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
|
|
FROM server_deployments d ORDER BY started_at DESC, rowid DESC`).all(REMOVING_SERVER);
|
|
}
|
|
|
|
export function interruptDeployments(): void {
|
|
db().transaction(() => {
|
|
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)").all() as { id: string }[];
|
|
const finishedAt = new Date().toISOString();
|
|
const error = 'The application restarted before this operation finished. Check the server before retrying.';
|
|
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)`).run(finishedAt, error);
|
|
for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome',
|
|
message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } });
|
|
})();
|
|
}
|