josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/history.ts
+551 -146
View File
@@ -4,52 +4,107 @@ import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js';
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js';
import type {
ProvisioningCapacity,
ProvisioningDeployment,
ProvisioningDetail,
ProvisioningSource
} from '../../provisioning-detail.js';
import { fleetSettings } from '../settings.js';
import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
import { bindNativeOperation } from './native.js';
import { workerActivity } from '../pipeline/activity.js';
import { canCancelCreation } from './cancel-creation.js';
import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js';
import {
canCancelProvisioningJob,
provisioningCancelled,
rebalanceCancellationGuard
} from './cancellation.js';
import { committedRebalanceStartup } from './concurrency.js';
import { nativeDeployment } from '../backends/native.js';
import { ensureHostSchema } from '../hosts/schema.js';
import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
import { serverTitle } from '../../ui/server-labels.js';
import { provisioningQueueStatus } from '../../ui/provisioning-queue.js';
import { provisioningPageSize, provisioningPageOffset, provisioningSourceKinds, type ProvisioningListOptions, type ProvisioningListPage, type ProvisioningListRow } from '../../provisioning-list.js';
import {
provisioningPageSize,
provisioningPageOffset,
provisioningSourceKinds,
type ProvisioningListOptions,
type ProvisioningListPage,
type ProvisioningListRow
} from '../../provisioning-list.js';
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
export function beginDeployment(
server: string,
action: ProvisioningAction,
triggeredBy = 'ui'
): string {
const id = randomUUID();
db().transaction(() => {
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by, requested_by)
VALUES (?, ?, ?, 'running', 'Starting', ?, ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy,
triggeredBy === 'ui' || triggeredBy.startsWith('rebalance:') ? currentRequestUsername() : null);
db()
.prepare(
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by, requested_by)
VALUES (?, ?, ?, 'running', 'Starting', ?, ?, ?)`
)
.run(
id,
server,
action,
new Date().toISOString(),
triggeredBy,
triggeredBy === 'ui' || triggeredBy.startsWith('rebalance:')
? currentRequestUsername()
: null
);
bindNativeOperation(id, server);
recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested',
label: action === 'create' ? 'Server creation requested'
: action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested` } });
recordProgress(id, {
phase: 'requested',
observedAt: null,
checkpoint: {
key: 'requested',
phase: 'requested',
label:
action === 'create'
? 'Server creation requested'
: action === 'destroy'
? 'Server destruction requested'
: `${actionLabels[action]} server requested`
}
});
})();
return id;
}
export function deploymentStage(id: string, stage: string, update?: ProgressUpdate): void {
db().transaction(() => {
const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
const previous = db()
.prepare('SELECT stage FROM server_deployments WHERE id = ?')
.get(id) as { stage: string } | undefined;
const changed = !!previous && previous.stage !== stage;
const before = update && changed ? activitySummary(id).latestId ?? 0 : 0;
const before = update && changed ? (activitySummary(id).latestId ?? 0) : 0;
db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
if (update) recordProgress(id, update);
// Structured transitions may already carry the exact legacy stage text.
const alreadyRecorded = update && changed && db().prepare(`SELECT 1 FROM server_deployment_events
WHERE deployment_id = ? AND id > ? AND message = ? LIMIT 1`).get(id, before, stage);
const alreadyRecorded =
update &&
changed &&
db()
.prepare(
`SELECT 1 FROM server_deployment_events
WHERE deployment_id = ? AND id > ? AND message = ? LIMIT 1`
)
.get(id, before, stage);
if (changed && !alreadyRecorded) recordDeploymentEvent(id, stage);
})();
}
export function recordDeploymentEvent(id: string, message: string): void {
db().prepare('INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)')
db()
.prepare(
'INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)'
)
.run(id, new Date().toISOString(), message);
}
@@ -58,103 +113,239 @@ function warningMessages(raw: string): string[] {
const parsed: unknown = JSON.parse(raw);
if (!Array.isArray(parsed)) return ['Recorded warnings could not be read.'];
const messages = parsed.filter((value): value is string => typeof value === 'string');
if (messages.length !== parsed.length) messages.push('Some recorded warnings could not be read.');
if (messages.length !== parsed.length)
messages.push('Some recorded warnings could not be read.');
return messages;
} catch { return ['Recorded warnings could not be read.']; }
} catch {
return ['Recorded warnings could not be read.'];
}
}
function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
const id = source.source_id;
if (source.kind === 'rebalance') {
const exists = !!db().prepare('SELECT 1 FROM rebalancing_runs WHERE id=?').get(id);
return { ...source, label: exists ? 'Rebalance run' : 'Rebalance history expired',
href: exists ? `/capacity?rebalance=1&rebalanceRun=${encodeURIComponent(id)}` : null };
return {
...source,
label: exists ? 'Rebalance run' : 'Rebalance history expired',
href: exists ? `/capacity?rebalance=1&rebalanceRun=${encodeURIComponent(id)}` : null
};
}
if (source.kind === 'autoscaling') {
const exists = !!db().prepare('SELECT 1 FROM autoscaling_policies WHERE id=?').get(id);
return { ...source, label: exists ? 'Autoscaling policy' : 'Deleted autoscaling policy', href: exists ? `/autoscaling#${encodeURIComponent(id)}` : null };
return {
...source,
label: exists ? 'Autoscaling policy' : 'Deleted autoscaling policy',
href: exists ? `/autoscaling#${encodeURIComponent(id)}` : null
};
}
if (source.kind === 'campaign') {
const row = db().prepare(`SELECT d.action, d.beta_id, s.name FROM deployments d
const row = db()
.prepare(
`SELECT d.action, d.beta_id, s.name FROM deployments d
LEFT JOIN scripts s ON s.id=COALESCE(d.script_id, (SELECT script_id FROM deployment_scripts WHERE deployment_id=d.id ORDER BY script_id LIMIT 1))
WHERE d.id=?`).get(id) as { action: string; beta_id: number | null; name: string | null } | undefined;
const action = row?.action === 'promote' ? 'Promotion' : row?.action === 'rollback' ? 'Rollback' : row?.action === 'drift' ? 'Maintenance repair' : 'Campaign';
return { ...source, label: row ? `${action}${row.beta_id ? ` · Beta #${row.beta_id}` : row.name ? ` · ${row.name}` : action === 'Campaign' ? '' : ' campaign'}` : 'Campaign unavailable', href: row ? `/campaigns/${encodeURIComponent(id)}` : null };
WHERE d.id=?`
)
.get(id) as { action: string; beta_id: number | null; name: string | null } | undefined;
const action =
row?.action === 'promote'
? 'Promotion'
: row?.action === 'rollback'
? 'Rollback'
: row?.action === 'drift'
? 'Maintenance repair'
: 'Campaign';
return {
...source,
label: row
? `${action}${row.beta_id ? ` · Beta #${row.beta_id}` : row.name ? ` · ${row.name}` : action === 'Campaign' ? '' : ' campaign'}`
: 'Campaign unavailable',
href: row ? `/campaigns/${encodeURIComponent(id)}` : null
};
}
if (source.kind === 'beta') {
const row = db().prepare('SELECT notes FROM betas WHERE id = ?').get(id) as { notes: string | null } | undefined;
const row = db().prepare('SELECT notes FROM betas WHERE id = ?').get(id) as
{ notes: string | null } | undefined;
const label = row?.notes?.trim().split('\n')[0].slice(0, 120);
return { ...source, label: label ? `Beta #${id}: ${label}` : `Beta #${id}`, href: row ? `/betas/${encodeURIComponent(id)}` : null };
return {
...source,
label: label ? `Beta #${id}: ${label}` : `Beta #${id}`,
href: row ? `/betas/${encodeURIComponent(id)}` : null
};
}
if (source.kind === 'watchdog') {
const row = db().prepare('SELECT summary FROM watchdog_incidents WHERE id = ?').get(id) as { summary: string } | undefined;
return { ...source, label: row?.summary ?? `Maintenance finding ${id.slice(0, 8)}`, href: row ? `/maintenance/${encodeURIComponent(id)}` : null };
const row = db().prepare('SELECT summary FROM watchdog_incidents WHERE id = ?').get(id) as
{ summary: string } | undefined;
return {
...source,
label: row?.summary ?? `Maintenance finding ${id.slice(0, 8)}`,
href: row ? `/maintenance/${encodeURIComponent(id)}` : null
};
}
const row = db().prepare('SELECT kind FROM maintenance_jobs WHERE id = ?').get(id) as { kind: string } | undefined;
return { ...source, label: row && row.kind !== 'legacy' ? `Maintenance ${row.kind} #${id}` : `Maintenance #${id}`,
href: row ? `/maintenance?view=activity#job-${encodeURIComponent(id)}` : null };
const row = db().prepare('SELECT kind FROM maintenance_jobs WHERE id = ?').get(id) as
{ kind: string } | undefined;
return {
...source,
label:
row && row.kind !== 'legacy' ? `Maintenance ${row.kind} #${id}` : `Maintenance #${id}`,
href: row ? `/maintenance?view=activity#job-${encodeURIComponent(id)}` : null
};
}
/** Active operations get current pool snapshots, not a second admission/eligibility engine. */
function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): ProvisioningCapacity {
if (!['queued', 'waiting', 'running', 'awaiting-health'].includes(deployment.state)) return { evaluatedAt, pools: [] };
function capacityFor(
deployment: ProvisioningDeployment,
evaluatedAt: string
): ProvisioningCapacity {
if (!['queued', 'waiting', 'running', 'awaiting-health'].includes(deployment.state))
return { evaluatedAt, pools: [] };
const settings = fleetSettings();
const pools: ProvisioningCapacity['pools'] = [];
if (deployment.action === 'create' || deployment.action === 'rebuild') {
const jobs = db().prepare(`SELECT id, server, state FROM server_deployments
const jobs = db()
.prepare(
`SELECT id, server, state FROM server_deployments
WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup})
ORDER BY started_at, rowid`).all() as { id: string; server: string; state: string }[];
const holders = jobs.filter(job => job.state !== 'queued').map(({ id, server }) => ({ id, server }));
const order = jobs.filter(job => job.state === 'queued').findIndex(job => job.id === deployment.id);
pools.push({ kind: 'startup', limit: settings.createConcurrency, used: holders.length, holders,
requestOrder: order < 0 ? null : order + 1, ownsSlot: holders.some(holder => holder.id === deployment.id) });
ORDER BY started_at, rowid`
)
.all() as { id: string; server: string; state: string }[];
const holders = jobs
.filter((job) => job.state !== 'queued')
.map(({ id, server }) => ({ id, server }));
const order = jobs
.filter((job) => job.state === 'queued')
.findIndex((job) => job.id === deployment.id);
pools.push({
kind: 'startup',
limit: settings.createConcurrency,
used: holders.length,
holders,
requestOrder: order < 0 ? null : order + 1,
ownsSlot: holders.some((holder) => holder.id === deployment.id)
});
}
const creationRecovery = deployment.action === 'create' && db().prepare(`SELECT 1 FROM native_provisioning
WHERE deployment_id=? AND recovery_step IS NOT NULL`).get(deployment.id);
const creationRecovery =
deployment.action === 'create' &&
db()
.prepare(
`SELECT 1 FROM native_provisioning
WHERE deployment_id=? AND recovery_step IS NOT NULL`
)
.get(deployment.id);
if (['start', 'restart', 'rebuild'].includes(deployment.action) || creationRecovery) {
const holders = db().prepare(`SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL
ORDER BY requested_at, server`).all() as { id: string | null; server: string }[];
pools.push({ kind: 'restart', limit: settings.restartConcurrency, used: holders.length, holders,
requestOrder: null, ownsSlot: holders.some(holder => holder.id === deployment.id) });
const holders = db()
.prepare(
`SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL
ORDER BY requested_at, server`
)
.all() as { id: string | null; server: string }[];
pools.push({
kind: 'restart',
limit: settings.restartConcurrency,
used: holders.length,
holders,
requestOrder: null,
ownsSlot: holders.some((holder) => holder.id === deployment.id)
});
}
return { evaluatedAt, pools };
}
/** The default retains the full legacy event response. Page clients opt into a bounded window. */
export function getDeployment(id: string, options: { events?: 'all' | 'recent' } = {}): ProvisioningDetail | null {
export function getDeployment(
id: string,
options: { events?: 'all' | 'recent' } = {}
): ProvisioningDetail | null {
return db().transaction(() => {
const row = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id) as
(ProvisioningDeployment & { observation_json: string | null; phase_key: string | null; phase_started_at: string | null; observed_at: string | null }) | undefined;
const row = db()
.prepare(
`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`
)
.get(id) as
| (ProvisioningDeployment & {
observation_json: string | null;
phase_key: string | null;
phase_started_at: string | null;
observed_at: string | null;
})
| undefined;
if (!row) return null;
const generatedAt = new Date().toISOString();
const native = nativeDeployment(id);
const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
const agentOperation = native
? (db()
.prepare(
`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
FROM native_provisioning n LEFT JOIN ec_host_migrations m ON m.deployment_id=n.deployment_id
JOIN ec_host_operations o ON o.id=COALESCE(n.operation_id,m.remove_operation_id) WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`).get(id) as ProvisioningDetail['cancellation'];
const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
if (row.triggered_by.startsWith('rebalance:')) sources.push(sourceDetails({ kind: 'rebalance', source_id: row.triggered_by.slice('rebalance:'.length) }));
JOIN ec_host_operations o ON o.id=COALESCE(n.operation_id,m.remove_operation_id) WHERE n.deployment_id=?`
)
.get(id) as ProvisioningDetail['agentOperation'])
: null;
const cancellation = db()
.prepare(
`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`
)
.get(id) as ProvisioningDetail['cancellation'];
const {
observation_json: _snapshot,
phase_key: _phase,
phase_started_at: _phaseStarted,
observed_at: _observed,
...deployment
} = row;
const sources = (
db()
.prepare(
'SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id'
)
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]
).map(sourceDetails);
if (row.triggered_by.startsWith('rebalance:'))
sources.push(
sourceDetails({
kind: 'rebalance',
source_id: row.triggered_by.slice('rebalance:'.length)
})
);
const launch = recipeSummaryForDeployment(id);
const collection = db().prepare(`SELECT c.id, c.name FROM (
const collection = db()
.prepare(
`SELECT c.id, c.name FROM (
SELECT collection_id, deployment_id FROM server_collection_members
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`
)
.get(id) as { id: string; name: string } | undefined;
const override = safetyOverrideAvailability(id);
return { deployment, backend: 'native' as const, agentOperation: agentOperation ?? null,
canOverrideSafety: override.applicable, safetyOverrideUnavailableReason: override.unavailableReason, safetyOverride: safetyOverride(id),
canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
canResumeRebuild: canResumeRebuild(id), canRetryWithRebuild: canRetryWithRebuild(id),
canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
warningMessages: warningMessages(deployment.warnings), generatedAt };
return {
deployment,
backend: 'native' as const,
agentOperation: agentOperation ?? null,
canOverrideSafety: override.applicable,
safetyOverrideUnavailableReason: override.unavailableReason,
safetyOverride: safetyOverride(id),
canCancel: canCancelProvisioningJob(id),
canCancelCreation: canCancelCreation(id),
cancellation: cancellation ?? null,
workerActivity: workerActivity(),
events: deploymentEvents(id, options.events === 'recent'),
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) },
sources,
recipe: launch?.recipe ?? null,
blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id),
canResumeCleanup: canResumeCleanup(id),
canResumeRebuild: canResumeRebuild(id),
canRetryWithRebuild: canRetryWithRebuild(id),
canCancelRemoval: canCancelRemoval(id),
collection: collection ?? null,
progress: readProgress(row),
capacity: capacityFor(deployment, generatedAt),
warningMessages: warningMessages(deployment.warnings),
generatedAt
};
})();
}
@@ -162,29 +353,46 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
export const REMOVING_SERVER = 'Removing server files and registry';
export function canCancelRemoval(id: string): boolean {
return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
return !!db()
.prepare(
"SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?"
)
.get(id, REMOVING_SERVER);
}
/** Competes atomically with the worker's claim, before any external removal. */
export function cancelRemoval(id: string, pausePolicy = true): boolean {
return db().transaction(() => {
const finishedAt = new Date().toISOString();
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(finishedAt, id, REMOVING_SERVER);
const result = db()
.prepare(
`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`
)
.run(finishedAt, id, REMOVING_SERVER);
if (!result.changes) return false;
if (pausePolicy) pauseScalingForOperation(id);
recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome',
message: 'Queued removal cancelled. The server and its files were kept.',
details: { state: 'cancelled', finishedAt } } });
recordProgress(id, {
blocker: null,
checks: [],
event: {
kind: 'outcome',
message: 'Queued removal cancelled. The server and its files were kept.',
details: { state: 'cancelled', finishedAt }
}
});
return true;
})();
}
export function canResumeCleanup(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d
return !!db()
.prepare(
`SELECT d.id FROM server_deployments d
WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server))
AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`
)
.get(id, REMOVING_SERVER);
}
@@ -192,16 +400,30 @@ export function resumeCleanup(id: string): boolean {
return db().transaction(() => {
if (!canResumeCleanup(id)) return false;
preserveOutcome(id);
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed instance removal' } });
db()
.prepare(
"UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?"
)
.run(id);
recordProgress(id, {
phase: 'cleanup',
blocker: null,
checks: [],
observedAt: null,
event: {
kind: 'resumed',
message: 'Resumed local cleanup after confirmed instance removal'
}
});
return true;
})();
}
export function canResumeVerification(id: string): boolean {
ensureHostSchema(db());
return !!db().prepare(`SELECT d.id FROM server_deployments d
return !!db()
.prepare(
`SELECT d.id FROM server_deployments d
JOIN server_creation_jobs j ON j.deployment_id = d.id
JOIN servers s ON s.name = j.server AND s.game_port = j.port AND s.destroyed_at IS NULL
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
@@ -215,14 +437,17 @@ export function canResumeVerification(id: string): boolean {
OR (o.kind='instance.rebuild' AND i.generation IN (n.generation,n.generation+1))))
OR (n.deployment_id IS NULL AND (d.action='create' OR EXISTS (
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id=d.id AND r.removed_at IS NOT NULL))))
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`
)
.get(id);
}
/** A conclusively failed initial create can enter the normal rebuild flow. */
export function canRetryWithRebuild(id: string): boolean {
ensureHostSchema(db());
return !!db().prepare(`SELECT d.id FROM server_deployments d
return !!db()
.prepare(
`SELECT d.id FROM server_deployments d
JOIN server_creation_jobs j ON j.deployment_id=d.id
JOIN servers s ON s.name=d.server AND s.game_port=j.port AND s.destroyed_at IS NULL
JOIN native_provisioning n ON n.deployment_id=d.id AND n.recovery_step IS NULL
@@ -233,14 +458,18 @@ export function canRetryWithRebuild(id: string): boolean {
WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action='create'
AND d.state IN ('failed','interrupted') AND o.kind='instance.create' AND o.state='failed'
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id)
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)`)
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)`
)
.get(id);
}
export function canResumeRebuild(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
return !!db()
.prepare(
`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`
)
.get(id, REMOVING_SERVER);
}
@@ -248,9 +477,21 @@ export function resumeRebuild(id: string): boolean {
return db().transaction(() => {
if (!canResumeRebuild(id)) return false;
preserveOutcome(id);
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed instance removal' } });
db()
.prepare(
"UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?"
)
.run(id);
recordProgress(id, {
phase: 'cleanup',
blocker: null,
checks: [],
observedAt: null,
event: {
kind: 'resumed',
message: 'Resumed rebuild cleanup after confirmed instance removal'
}
});
return true;
})();
}
@@ -260,46 +501,118 @@ export function resumeVerification(id: string): boolean {
return db().transaction(() => {
if (!canResumeVerification(id)) return false;
preserveOutcome(id);
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for host agent health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
event: { kind: 'resumed', message: 'Resumed verification of the existing server' } });
db()
.prepare(
"UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?"
)
.run(id);
deploymentStage(id, 'Waiting for host agent health', {
phase: 'verify-container',
blocker: null,
checks: [],
observedAt: null,
event: { kind: 'resumed', message: 'Resumed verification of the existing server' }
});
return true;
})();
}
export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
export function finishDeployment(
id: string,
result: { ok: boolean; message?: string; warnings?: string[] }
): void {
// Creation failure cleanup deletes its own registry row, whose history trigger
// marks it cancelled. Preserve that real failure; explicit cancellation stays terminal.
if (provisioningCancelled(id) && !db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NOT NULL AND error='Server deleted'").get(id)) return;
if (
provisioningCancelled(id) &&
!db()
.prepare(
"SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NOT NULL AND error='Server deleted'"
)
.get(id)
)
return;
const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
db().transaction(() => {
const finishedAt = new Date().toISOString();
db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
.run(state, finishedAt, result.message ?? null, JSON.stringify(result.warnings ?? []), id);
for (const warning of result.warnings ?? []) recordProgress(id, { event: { kind: 'warning', message: warning } });
recordProgress(id, { blocker: null, event: { kind: 'outcome',
message: result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`,
details: { state, finishedAt, error: result.message ?? null, warnings: result.warnings ?? [] } } });
db()
.prepare(
`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`
)
.run(
state,
finishedAt,
result.message ?? null,
JSON.stringify(result.warnings ?? []),
id
);
for (const warning of result.warnings ?? [])
recordProgress(id, { event: { kind: 'warning', message: warning } });
recordProgress(id, {
blocker: null,
event: {
kind: 'outcome',
message: result.ok
? state === 'warning'
? 'Completed with warnings'
: 'Completed'
: `Failed: ${result.message ?? 'Unknown error'}`,
details: {
state,
finishedAt,
error: result.message ?? null,
warnings: result.warnings ?? []
}
}
});
})();
}
/** Legacy failures also need a durable structured result before same-ID resume clears it. */
function preserveOutcome(id: string): void {
const row = db().prepare('SELECT state, finished_at, error, warnings, observation_json FROM server_deployments WHERE id = ?').get(id) as
{ state: string; finished_at: string | null; error: string | null; warnings: string; observation_json: string | null } | undefined;
if (!row || !['failed', 'interrupted', 'warning', 'done', 'cancelled'].includes(row.state)) return;
const previous = db().prepare("SELECT details_json FROM server_deployment_events WHERE deployment_id = ? AND kind = 'outcome' ORDER BY id DESC LIMIT 1")
const row = db()
.prepare(
'SELECT state, finished_at, error, warnings, observation_json FROM server_deployments WHERE id = ?'
)
.get(id) as
| {
state: string;
finished_at: string | null;
error: string | null;
warnings: string;
observation_json: string | null;
}
| undefined;
if (!row || !['failed', 'interrupted', 'warning', 'done', 'cancelled'].includes(row.state))
return;
const previous = db()
.prepare(
"SELECT details_json FROM server_deployment_events WHERE deployment_id = ? AND kind = 'outcome' ORDER BY id DESC LIMIT 1"
)
.get(id) as { details_json: string | null } | undefined;
try {
const details = JSON.parse(previous?.details_json ?? 'null');
if (details?.state === row.state && details?.finishedAt === row.finished_at) return;
} catch { /* Preserve the row's known outcome when old event metadata is missing. */ }
recordProgress(id, { observedAt: readProgress(row)?.observedAt ?? null, event: { kind: 'outcome',
message: `${row.state === 'interrupted' ? 'Interrupted' : 'Recorded outcome'}: ${row.error ?? row.state}`,
details: { state: row.state, finishedAt: row.finished_at, error: row.error, warnings: warningMessages(row.warnings) } } });
} catch {
/* Preserve the row's known outcome when old event metadata is missing. */
}
recordProgress(id, {
observedAt: readProgress(row)?.observedAt ?? null,
event: {
kind: 'outcome',
message: `${row.state === 'interrupted' ? 'Interrupted' : 'Recorded outcome'}: ${row.error ?? row.state}`,
details: {
state: row.state,
finishedAt: row.finished_at,
error: row.error,
warnings: warningMessages(row.warnings)
}
}
});
}
const activeDeployment = "d.server_deleted_at IS NULL AND d.state IN ('queued', 'awaiting-health', 'running', 'waiting')";
const activeDeployment =
"d.server_deleted_at IS NULL AND d.state IN ('queued', 'awaiting-health', 'running', 'waiting')";
const reviewDeployment = `d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS (
SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=d.id
AND c.resolved_at >= COALESCE(d.finished_at,d.started_at))`;
@@ -314,29 +627,53 @@ const deploymentListColumns = `d.id, d.server, d.server_deleted_at, d.action, d.
`;
function deploymentListQuery(options: ProvisioningListOptions) {
const conditions = [options.view === 'active' ? `(${activeDeployment})`
: options.view === 'history' ? `NOT (${activeDeployment})` : '1'];
const conditions = [
options.view === 'active'
? `(${activeDeployment})`
: options.view === 'history'
? `NOT (${activeDeployment})`
: '1'
];
const parameters: string[] = [];
if (options.action) { conditions.push('d.action = ?'); parameters.push(options.action); }
if (options.action) {
conditions.push('d.action = ?');
parameters.push(options.action);
}
if (options.status === 'attention') conditions.push(reviewDeployment);
else if (options.status === 'pending') conditions.push("d.state IN ('queued','waiting')");
else if (options.status) { conditions.push('d.state=?'); parameters.push(options.status); }
const linked = (kinds: readonly string[]) => `(EXISTS (SELECT 1 FROM provisioning_sources src WHERE src.operation_id=d.id AND src.kind IN (${kinds.map(kind => `'${kind}'`).join(',')}))
OR ${kinds.map(kind => `d.triggered_by LIKE '${kind}:%'`).join(' OR ')})`;
else if (options.status) {
conditions.push('d.state=?');
parameters.push(options.status);
}
const linked = (
kinds: readonly string[]
) => `(EXISTS (SELECT 1 FROM provisioning_sources src WHERE src.operation_id=d.id AND src.kind IN (${kinds.map((kind) => `'${kind}'`).join(',')}))
OR ${kinds.map((kind) => `d.triggered_by LIKE '${kind}:%'`).join(' OR ')})`;
if (options.source === 'manual' || options.source === 'system') {
conditions.push(`NOT ${linked(Object.values(provisioningSourceKinds).flat())}`);
conditions.push(options.source === 'manual' ? "d.triggered_by='ui'" : "d.triggered_by!='ui'");
conditions.push(
options.source === 'manual' ? "d.triggered_by='ui'" : "d.triggered_by!='ui'"
);
} else if (options.source) conditions.push(linked(provisioningSourceKinds[options.source]));
const search = (options.server ?? '').slice(0, 100).toLowerCase();
if (search) {
// Read only distinct names to preserve friendly map/lobby searches, including deleted servers.
const names = (db().prepare('SELECT DISTINCT server FROM server_deployments').all() as { server: string }[])
.map(row => row.server).filter(name => `${name} ${serverTitle(name)}`.toLowerCase().includes(search));
const names = (
db().prepare('SELECT DISTINCT server FROM server_deployments').all() as {
server: string;
}[]
)
.map((row) => row.server)
.filter((name) => `${name} ${serverTitle(name)}`.toLowerCase().includes(search));
conditions.push('d.server IN (SELECT value FROM json_each(?))');
parameters.push(JSON.stringify(names));
}
return { where: conditions.join(' AND '), parameters,
limit: provisioningPageSize(options.limit ?? 25), offset: provisioningPageOffset(options.offset ?? 0) };
return {
where: conditions.join(' AND '),
parameters,
limit: provisioningPageSize(options.limit ?? 25),
offset: provisioningPageOffset(options.offset ?? 0)
};
}
function activeDeploymentRank(row: ProvisioningListRow): number {
@@ -346,32 +683,72 @@ function activeDeploymentRank(row: ProvisioningListRow): number {
// Launch preparation keeps its queued/waiting state until command submission.
if (row.phase === 'prepare' && !row.waitCode) return 1;
// Remote file staging currently records its progress only in the stage message.
if (row.waitCode === 'native-host' && ['Staging and verifying files on the host.',
'Waiting for verified remote file staging.'].includes(row.stage)) return 1;
if (
row.waitCode === 'native-host' &&
[
'Staging and verifying files on the host.',
'Waiting for verified remote file staging.'
].includes(row.stage)
)
return 1;
if (group === 'queued') return 3;
// A submitted command remains in progress while its agent result is pending.
if (row.state === 'running' && row.phase === 'container-action' && row.waitCode === 'native-host') return 1;
if (
row.state === 'running' &&
row.phase === 'container-action' &&
row.waitCode === 'native-host'
)
return 1;
return 2;
}
function deploymentListRows(query: ReturnType<typeof deploymentListQuery>, active: boolean): ProvisioningListRow[] {
const rows = db().prepare(`SELECT ${deploymentListColumns} FROM server_deployments d WHERE ${query.where}
ORDER BY ${active ? 'd.started_at ASC, d.rowid ASC' : 'd.started_at DESC, d.rowid DESC LIMIT ? OFFSET ?'}`)
.all(REMOVING_SERVER, ...query.parameters, ...(active ? [] : [query.limit, query.offset])) as ProvisioningListRow[];
function deploymentListRows(
query: ReturnType<typeof deploymentListQuery>,
active: boolean
): ProvisioningListRow[] {
const rows = db()
.prepare(
`SELECT ${deploymentListColumns} FROM server_deployments d WHERE ${query.where}
ORDER BY ${active ? 'd.started_at ASC, d.rowid ASC' : 'd.started_at DESC, d.rowid DESC LIMIT ? OFFSET ?'}`
)
.all(
REMOVING_SERVER,
...query.parameters,
...(active ? [] : [query.limit, query.offset])
) as ProvisioningListRow[];
if (!rows.length) return rows;
// Stable sorting keeps request/insertion order within each progress group.
if (active) rows.sort((left, right) => activeDeploymentRank(left) - activeDeploymentRank(right));
const links = db().prepare(`SELECT operation_id,kind,source_id FROM provisioning_sources
WHERE operation_id IN (SELECT value FROM json_each(?)) ORDER BY kind,source_id`).all(JSON.stringify(rows.map(row => row.id))) as
(Omit<ProvisioningSource, 'label' | 'href'> & { operation_id: string })[];
if (active)
rows.sort((left, right) => activeDeploymentRank(left) - activeDeploymentRank(right));
const links = db()
.prepare(
`SELECT operation_id,kind,source_id FROM provisioning_sources
WHERE operation_id IN (SELECT value FROM json_each(?)) ORDER BY kind,source_id`
)
.all(JSON.stringify(rows.map((row) => row.id))) as (Omit<
ProvisioningSource,
'label' | 'href'
> & { operation_id: string })[];
const byOperation = new Map<string, Omit<ProvisioningSource, 'label' | 'href'>[]>();
for (const { operation_id, ...link } of links) byOperation.set(operation_id, [...(byOperation.get(operation_id) ?? []), link]);
for (const { operation_id, ...link } of links)
byOperation.set(operation_id, [...(byOperation.get(operation_id) ?? []), link]);
const cache = new Map<string, ProvisioningSource>();
for (const row of rows) {
const sources = byOperation.get(row.id) ?? [];
const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling|rebalance):(.+)$/);
if (fallback && !sources.some(source => source.kind === fallback[1] && source.source_id === fallback[2])) sources.unshift({ kind: fallback[1] as ProvisioningSource['kind'], source_id: fallback[2] });
row.sources = sources.map(source => {
const fallback = row.triggered_by.match(
/^(campaign|beta|maintenance|watchdog|autoscaling|rebalance):(.+)$/
);
if (
fallback &&
!sources.some(
(source) => source.kind === fallback[1] && source.source_id === fallback[2]
)
)
sources.unshift({
kind: fallback[1] as ProvisioningSource['kind'],
source_id: fallback[2]
});
row.sources = sources.map((source) => {
const key = `${source.kind}:${source.source_id}`;
if (!cache.has(key)) cache.set(key, sourceDetails(source));
return cache.get(key)!;
@@ -389,10 +766,20 @@ export function deploymentListPage(options: ProvisioningListOptions = {}): Provi
return db().transaction(() => {
const query = deploymentListQuery(options);
const deployments = deploymentListRows(query, options.view === 'active');
const total = options.view === 'active' ? deployments.length : (db().prepare(
`SELECT COUNT(*) AS total FROM server_deployments d WHERE ${query.where}`
).get(...query.parameters) as { total: number }).total;
const { needsReview } = db().prepare(`SELECT COUNT(*) AS needsReview FROM server_deployments d WHERE ${reviewDeployment}`)
const total =
options.view === 'active'
? deployments.length
: (
db()
.prepare(
`SELECT COUNT(*) AS total FROM server_deployments d WHERE ${query.where}`
)
.get(...query.parameters) as { total: number }
).total;
const { needsReview } = db()
.prepare(
`SELECT COUNT(*) AS needsReview FROM server_deployments d WHERE ${reviewDeployment}`
)
.get() as { needsReview: number };
return { deployments, total, needsReview, limit: query.limit, offset: query.offset };
})();
@@ -400,12 +787,30 @@ export function deploymentListPage(options: ProvisioningListOptions = {}): Provi
export function interruptDeployments(): void {
db().transaction(() => {
const native = 'SELECT deployment_id FROM native_provisioning UNION ALL SELECT deployment_id FROM native_creation_intents';
const running = db().prepare(`SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (${native})`).all() as { id: string }[];
const native =
'SELECT deployment_id FROM native_provisioning UNION ALL SELECT deployment_id FROM native_creation_intents';
const running = db()
.prepare(
`SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (${native})`
)
.all() as { id: string }[];
const finishedAt = new Date().toISOString();
const error = 'The application restarted before this operation finished. Check the server before retrying.';
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (${native})`).run(finishedAt, error);
for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome',
message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } });
const error =
'The application restarted before this operation finished. Check the server before retrying.';
db()
.prepare(
`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (${native})`
)
.run(finishedAt, error);
for (const { id } of running)
recordProgress(id, {
blocker: null,
event: {
kind: 'outcome',
message:
'Interrupted by application restart. Check the server before retrying.',
details: { state: 'interrupted', finishedAt, error }
}
});
})();
}