795 lines
33 KiB
TypeScript
795 lines
33 KiB
TypeScript
import { pauseScalingForOperation } from '../autoscaling/ownership.js';
|
|
import { currentRequestUsername } from '../auth/request-context.js';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { db } from '../db/index.js';
|
|
import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js';
|
|
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
|
|
import type {
|
|
ProvisioningCapacity,
|
|
ProvisioningDeployment,
|
|
ProvisioningDetail,
|
|
ProvisioningSource
|
|
} from '../../provisioning-detail.js';
|
|
import { fleetSettings } from '../settings.js';
|
|
import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
|
|
import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
|
|
import { bindNativeOperation } from './native.js';
|
|
import { workerActivity } from '../pipeline/activity.js';
|
|
import { canCancelCreation } from './cancel-creation.js';
|
|
import {
|
|
canCancelProvisioningJob,
|
|
provisioningCancelled,
|
|
rebalanceCancellationGuard
|
|
} from './cancellation.js';
|
|
import { committedRebalanceStartup } from './concurrency.js';
|
|
import { nativeDeployment } from '../backends/native.js';
|
|
import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
|
|
import { serverTitle } from '../../ui/server-labels.js';
|
|
import {
|
|
provisioningPageSize,
|
|
provisioningPageOffset,
|
|
provisioningSourceKinds,
|
|
type ProvisioningListOptions,
|
|
type ProvisioningListPage,
|
|
type ProvisioningListRow
|
|
} from '../../provisioning-list.js';
|
|
|
|
export function beginDeployment(
|
|
server: string,
|
|
action: ProvisioningAction,
|
|
triggeredBy = 'ui'
|
|
): string {
|
|
const id = randomUUID();
|
|
db().transaction(() => {
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by, requested_by)
|
|
VALUES (?, ?, ?, 'running', 'Starting', ?, ?, ?)`
|
|
)
|
|
.run(
|
|
id,
|
|
server,
|
|
action,
|
|
new Date().toISOString(),
|
|
triggeredBy,
|
|
triggeredBy === 'ui' || triggeredBy.startsWith('rebalance:')
|
|
? currentRequestUsername()
|
|
: null
|
|
);
|
|
bindNativeOperation(id, server);
|
|
recordProgress(id, {
|
|
phase: 'requested',
|
|
observedAt: null,
|
|
checkpoint: {
|
|
key: 'requested',
|
|
phase: 'requested',
|
|
label:
|
|
action === 'create'
|
|
? 'Server creation requested'
|
|
: action === 'destroy'
|
|
? 'Server destruction requested'
|
|
: `${actionLabels[action]} server requested`
|
|
}
|
|
});
|
|
})();
|
|
return id;
|
|
}
|
|
|
|
export function deploymentStage(id: string, stage: string, update?: ProgressUpdate): void {
|
|
db().transaction(() => {
|
|
const previous = db()
|
|
.prepare('SELECT stage FROM server_deployments WHERE id = ?')
|
|
.get(id) as { stage: string } | undefined;
|
|
const changed = !!previous && previous.stage !== stage;
|
|
const before = update && changed ? (activitySummary(id).latestId ?? 0) : 0;
|
|
db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
|
|
if (update) recordProgress(id, update);
|
|
// Structured transitions may already carry the exact legacy stage text.
|
|
const alreadyRecorded =
|
|
update &&
|
|
changed &&
|
|
db()
|
|
.prepare(
|
|
`SELECT 1 FROM server_deployment_events
|
|
WHERE deployment_id = ? AND id > ? AND message = ? LIMIT 1`
|
|
)
|
|
.get(id, before, stage);
|
|
if (changed && !alreadyRecorded) recordDeploymentEvent(id, stage);
|
|
})();
|
|
}
|
|
|
|
export function recordDeploymentEvent(id: string, message: string): void {
|
|
db()
|
|
.prepare(
|
|
'INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)'
|
|
)
|
|
.run(id, new Date().toISOString(), message);
|
|
}
|
|
|
|
function warningMessages(raw: string): string[] {
|
|
try {
|
|
const parsed: unknown = JSON.parse(raw);
|
|
if (!Array.isArray(parsed)) return ['Recorded warnings could not be read.'];
|
|
const messages = parsed.filter((value): value is string => typeof value === 'string');
|
|
if (messages.length !== parsed.length)
|
|
messages.push('Some recorded warnings could not be read.');
|
|
return messages;
|
|
} catch {
|
|
return ['Recorded warnings could not be read.'];
|
|
}
|
|
}
|
|
|
|
function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
|
|
const id = source.source_id;
|
|
if (source.kind === 'rebalance') {
|
|
const exists = !!db().prepare('SELECT 1 FROM rebalancing_runs WHERE id=?').get(id);
|
|
return {
|
|
...source,
|
|
label: exists ? 'Rebalance run' : 'Rebalance history expired',
|
|
href: exists ? `/capacity?rebalance=1&rebalanceRun=${encodeURIComponent(id)}` : null
|
|
};
|
|
}
|
|
if (source.kind === 'autoscaling') {
|
|
const exists = !!db().prepare('SELECT 1 FROM autoscaling_policies WHERE id=?').get(id);
|
|
return {
|
|
...source,
|
|
label: exists ? 'Autoscaling policy' : 'Deleted autoscaling policy',
|
|
href: exists ? `/autoscaling#${encodeURIComponent(id)}` : null
|
|
};
|
|
}
|
|
if (source.kind === 'campaign') {
|
|
const row = db()
|
|
.prepare(
|
|
`SELECT d.action, d.beta_id, s.name FROM deployments d
|
|
LEFT JOIN scripts s ON s.id=COALESCE(d.script_id, (SELECT script_id FROM deployment_scripts WHERE deployment_id=d.id ORDER BY script_id LIMIT 1))
|
|
WHERE d.id=?`
|
|
)
|
|
.get(id) as { action: string; beta_id: number | null; name: string | null } | undefined;
|
|
const action =
|
|
row?.action === 'promote'
|
|
? 'Promotion'
|
|
: row?.action === 'rollback'
|
|
? 'Rollback'
|
|
: row?.action === 'drift'
|
|
? 'Maintenance repair'
|
|
: 'Campaign';
|
|
return {
|
|
...source,
|
|
label: row
|
|
? `${action}${row.beta_id ? ` · Beta #${row.beta_id}` : row.name ? ` · ${row.name}` : action === 'Campaign' ? '' : ' campaign'}`
|
|
: 'Campaign unavailable',
|
|
href: row ? `/campaigns/${encodeURIComponent(id)}` : null
|
|
};
|
|
}
|
|
if (source.kind === 'beta') {
|
|
const row = db().prepare('SELECT notes FROM betas WHERE id = ?').get(id) as
|
|
{ notes: string | null } | undefined;
|
|
const label = row?.notes?.trim().split('\n')[0].slice(0, 120);
|
|
return {
|
|
...source,
|
|
label: label ? `Beta #${id}: ${label}` : `Beta #${id}`,
|
|
href: row ? `/betas/${encodeURIComponent(id)}` : null
|
|
};
|
|
}
|
|
if (source.kind === 'watchdog') {
|
|
const row = db().prepare('SELECT summary FROM watchdog_incidents WHERE id = ?').get(id) as
|
|
{ summary: string } | undefined;
|
|
return {
|
|
...source,
|
|
label: row?.summary ?? `Maintenance finding ${id.slice(0, 8)}`,
|
|
href: row ? `/maintenance/${encodeURIComponent(id)}` : null
|
|
};
|
|
}
|
|
const row = db().prepare('SELECT kind FROM maintenance_jobs WHERE id = ?').get(id) as
|
|
{ kind: string } | undefined;
|
|
return {
|
|
...source,
|
|
label:
|
|
row && row.kind !== 'legacy' ? `Maintenance ${row.kind} #${id}` : `Maintenance #${id}`,
|
|
href: row ? `/maintenance?view=activity#job-${encodeURIComponent(id)}` : null
|
|
};
|
|
}
|
|
|
|
/** Active operations get current pool snapshots, not a second admission/eligibility engine. */
|
|
function capacityFor(
|
|
deployment: ProvisioningDeployment,
|
|
evaluatedAt: string
|
|
): ProvisioningCapacity {
|
|
if (!['queued', 'waiting', 'running', 'awaiting-health'].includes(deployment.state))
|
|
return { evaluatedAt, pools: [] };
|
|
const settings = fleetSettings();
|
|
const pools: ProvisioningCapacity['pools'] = [];
|
|
if (deployment.action === 'create' || deployment.action === 'rebuild') {
|
|
const jobs = db()
|
|
.prepare(
|
|
`SELECT id, server, state FROM server_deployments
|
|
WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup})
|
|
ORDER BY started_at, rowid`
|
|
)
|
|
.all() as { id: string; server: string; state: string }[];
|
|
const holders = jobs
|
|
.filter((job) => job.state !== 'queued')
|
|
.map(({ id, server }) => ({ id, server }));
|
|
const order = jobs
|
|
.filter((job) => job.state === 'queued')
|
|
.findIndex((job) => job.id === deployment.id);
|
|
pools.push({
|
|
kind: 'startup',
|
|
limit: settings.createConcurrency,
|
|
used: holders.length,
|
|
holders,
|
|
requestOrder: order < 0 ? null : order + 1,
|
|
ownsSlot: holders.some((holder) => holder.id === deployment.id)
|
|
});
|
|
}
|
|
const creationRecovery =
|
|
deployment.action === 'create' &&
|
|
db()
|
|
.prepare(
|
|
`SELECT 1 FROM native_provisioning
|
|
WHERE deployment_id=? AND recovery_step IS NOT NULL`
|
|
)
|
|
.get(deployment.id);
|
|
if (['start', 'restart', 'rebuild'].includes(deployment.action) || creationRecovery) {
|
|
const holders = db()
|
|
.prepare(
|
|
`SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL
|
|
ORDER BY requested_at, server`
|
|
)
|
|
.all() as { id: string | null; server: string }[];
|
|
pools.push({
|
|
kind: 'restart',
|
|
limit: settings.restartConcurrency,
|
|
used: holders.length,
|
|
holders,
|
|
requestOrder: null,
|
|
ownsSlot: holders.some((holder) => holder.id === deployment.id)
|
|
});
|
|
}
|
|
return { evaluatedAt, pools };
|
|
}
|
|
|
|
/** The default retains the full legacy event response. Page clients opt into a bounded window. */
|
|
export function getDeployment(
|
|
id: string,
|
|
options: { events?: 'all' | 'recent' } = {}
|
|
): ProvisioningDetail | null {
|
|
return db().transaction(() => {
|
|
const row = db()
|
|
.prepare(
|
|
`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
|
|
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`
|
|
)
|
|
.get(id) as
|
|
| (ProvisioningDeployment & {
|
|
observation_json: string | null;
|
|
phase_key: string | null;
|
|
phase_started_at: string | null;
|
|
observed_at: string | null;
|
|
})
|
|
| undefined;
|
|
if (!row) return null;
|
|
const generatedAt = new Date().toISOString();
|
|
const native = nativeDeployment(id);
|
|
const agentOperation = native
|
|
? (db()
|
|
.prepare(
|
|
`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
|
|
FROM native_provisioning n LEFT JOIN ec_host_migrations m ON m.deployment_id=n.deployment_id
|
|
JOIN ec_host_operations o ON o.id=COALESCE(n.operation_id,m.remove_operation_id) WHERE n.deployment_id=?`
|
|
)
|
|
.get(id) as ProvisioningDetail['agentOperation'])
|
|
: null;
|
|
const cancellation = db()
|
|
.prepare(
|
|
`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
|
|
JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`
|
|
)
|
|
.get(id) as ProvisioningDetail['cancellation'];
|
|
const {
|
|
observation_json: _snapshot,
|
|
phase_key: _phase,
|
|
phase_started_at: _phaseStarted,
|
|
observed_at: _observed,
|
|
...deployment
|
|
} = row;
|
|
const sources = (
|
|
db()
|
|
.prepare(
|
|
'SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id'
|
|
)
|
|
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]
|
|
).map(sourceDetails);
|
|
if (row.triggered_by.startsWith('rebalance:'))
|
|
sources.push(
|
|
sourceDetails({
|
|
kind: 'rebalance',
|
|
source_id: row.triggered_by.slice('rebalance:'.length)
|
|
})
|
|
);
|
|
const launch = recipeSummaryForDeployment(id);
|
|
const collection = db()
|
|
.prepare(
|
|
`SELECT c.id, c.name FROM (
|
|
SELECT collection_id, deployment_id FROM server_collection_members
|
|
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
|
|
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`
|
|
)
|
|
.get(id) as { id: string; name: string } | undefined;
|
|
const override = safetyOverrideAvailability(id);
|
|
return {
|
|
deployment,
|
|
backend: 'native' as const,
|
|
agentOperation: agentOperation ?? null,
|
|
canOverrideSafety: override.applicable,
|
|
safetyOverrideUnavailableReason: override.unavailableReason,
|
|
safetyOverride: safetyOverride(id),
|
|
canCancel: canCancelProvisioningJob(id),
|
|
canCancelCreation: canCancelCreation(id),
|
|
cancellation: cancellation ?? null,
|
|
workerActivity: workerActivity(),
|
|
events: deploymentEvents(id, options.events === 'recent'),
|
|
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) },
|
|
sources,
|
|
recipe: launch?.recipe ?? null,
|
|
blueprintName: launch?.blueprintName ?? null,
|
|
canResumeVerification: canResumeVerification(id),
|
|
canResumeCleanup: canResumeCleanup(id),
|
|
canResumeRebuild: canResumeRebuild(id),
|
|
canRetryWithRebuild: canRetryWithRebuild(id),
|
|
canCancelRemoval: canCancelRemoval(id),
|
|
collection: collection ?? null,
|
|
progress: readProgress(row),
|
|
capacity: capacityFor(deployment, generatedAt),
|
|
warningMessages: warningMessages(deployment.warnings),
|
|
generatedAt
|
|
};
|
|
})();
|
|
}
|
|
|
|
// This stage is written only after the host agent has confirmed instance removal.
|
|
export const REMOVING_SERVER = 'Removing server files and registry';
|
|
|
|
export function canCancelRemoval(id: string): boolean {
|
|
return !!db()
|
|
.prepare(
|
|
"SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?"
|
|
)
|
|
.get(id, REMOVING_SERVER);
|
|
}
|
|
|
|
/** Competes atomically with the worker's claim, before any external removal. */
|
|
export function cancelRemoval(id: string, pausePolicy = true): boolean {
|
|
return db().transaction(() => {
|
|
const finishedAt = new Date().toISOString();
|
|
const result = db()
|
|
.prepare(
|
|
`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
|
|
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`
|
|
)
|
|
.run(finishedAt, id, REMOVING_SERVER);
|
|
if (!result.changes) return false;
|
|
if (pausePolicy) pauseScalingForOperation(id);
|
|
recordProgress(id, {
|
|
blocker: null,
|
|
checks: [],
|
|
event: {
|
|
kind: 'outcome',
|
|
message: 'Queued removal cancelled. The server and its files were kept.',
|
|
details: { state: 'cancelled', finishedAt }
|
|
}
|
|
});
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
export function canResumeCleanup(id: string): boolean {
|
|
return !!db()
|
|
.prepare(
|
|
`SELECT d.id FROM server_deployments d
|
|
WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server))
|
|
AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`
|
|
)
|
|
.get(id, REMOVING_SERVER);
|
|
}
|
|
|
|
export function resumeCleanup(id: string): boolean {
|
|
return db().transaction(() => {
|
|
if (!canResumeCleanup(id)) return false;
|
|
preserveOutcome(id);
|
|
db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?"
|
|
)
|
|
.run(id);
|
|
recordProgress(id, {
|
|
phase: 'cleanup',
|
|
blocker: null,
|
|
checks: [],
|
|
observedAt: null,
|
|
event: {
|
|
kind: 'resumed',
|
|
message: 'Resumed local cleanup after confirmed instance removal'
|
|
}
|
|
});
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
export function canResumeVerification(id: string): boolean {
|
|
return !!db()
|
|
.prepare(
|
|
`SELECT d.id FROM server_deployments d
|
|
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
|
JOIN servers s ON s.name = j.server AND s.game_port = j.port AND s.destroyed_at IS NULL
|
|
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
|
LEFT JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
|
|
LEFT JOIN ec_host_operations o ON o.id=n.operation_id
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('create','rebuild')
|
|
AND d.state IN ('failed', 'interrupted')
|
|
AND ((n.deployment_id IS NOT NULL AND i.port=j.port AND o.state='completed'
|
|
AND o.instance_id=i.id AND o.host_id=i.host_id AND o.generation=n.generation
|
|
AND ((o.kind IN ('instance.create','instance.restart') AND i.generation=n.generation)
|
|
OR (o.kind='instance.rebuild' AND i.generation IN (n.generation,n.generation+1))))
|
|
OR (n.deployment_id IS NULL AND (d.action='create' OR EXISTS (
|
|
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id=d.id AND r.removed_at IS NOT NULL))))
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`
|
|
)
|
|
.get(id);
|
|
}
|
|
|
|
/** A conclusively failed initial create can enter the normal rebuild flow. */
|
|
export function canRetryWithRebuild(id: string): boolean {
|
|
return !!db()
|
|
.prepare(
|
|
`SELECT d.id FROM server_deployments d
|
|
JOIN server_creation_jobs j ON j.deployment_id=d.id
|
|
JOIN servers s ON s.name=d.server AND s.game_port=j.port AND s.destroyed_at IS NULL
|
|
JOIN native_provisioning n ON n.deployment_id=d.id AND n.recovery_step IS NULL
|
|
JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
|
|
AND i.generation=n.generation AND i.port=j.port
|
|
JOIN ec_host_operations o ON o.id=n.operation_id AND o.instance_id=i.id
|
|
AND o.host_id=i.host_id AND o.generation=n.generation
|
|
WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action='create'
|
|
AND d.state IN ('failed','interrupted') AND o.kind='instance.create' AND o.state='failed'
|
|
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id)
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)`
|
|
)
|
|
.get(id);
|
|
}
|
|
|
|
export function canResumeRebuild(id: string): boolean {
|
|
return !!db()
|
|
.prepare(
|
|
`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`
|
|
)
|
|
.get(id, REMOVING_SERVER);
|
|
}
|
|
|
|
export function resumeRebuild(id: string): boolean {
|
|
return db().transaction(() => {
|
|
if (!canResumeRebuild(id)) return false;
|
|
preserveOutcome(id);
|
|
db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?"
|
|
)
|
|
.run(id);
|
|
recordProgress(id, {
|
|
phase: 'cleanup',
|
|
blocker: null,
|
|
checks: [],
|
|
observedAt: null,
|
|
event: {
|
|
kind: 'resumed',
|
|
message: 'Resumed rebuild cleanup after confirmed instance removal'
|
|
}
|
|
});
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
/** Resume readiness after a completed command; unhealthy recovery still applies. */
|
|
export function resumeVerification(id: string): boolean {
|
|
return db().transaction(() => {
|
|
if (!canResumeVerification(id)) return false;
|
|
preserveOutcome(id);
|
|
db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?"
|
|
)
|
|
.run(id);
|
|
deploymentStage(id, 'Waiting for host agent health', {
|
|
phase: 'verify-container',
|
|
blocker: null,
|
|
checks: [],
|
|
observedAt: null,
|
|
event: { kind: 'resumed', message: 'Resumed verification of the existing server' }
|
|
});
|
|
return true;
|
|
})();
|
|
}
|
|
|
|
export function finishDeployment(
|
|
id: string,
|
|
result: { ok: boolean; message?: string; warnings?: string[] }
|
|
): void {
|
|
// Creation failure cleanup deletes its own registry row, whose history trigger
|
|
// marks it cancelled. Preserve that real failure; explicit cancellation stays terminal.
|
|
if (
|
|
provisioningCancelled(id) &&
|
|
!db()
|
|
.prepare(
|
|
"SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NOT NULL AND error='Server deleted'"
|
|
)
|
|
.get(id)
|
|
)
|
|
return;
|
|
const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
|
|
db().transaction(() => {
|
|
const finishedAt = new Date().toISOString();
|
|
db()
|
|
.prepare(
|
|
`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`
|
|
)
|
|
.run(
|
|
state,
|
|
finishedAt,
|
|
result.message ?? null,
|
|
JSON.stringify(result.warnings ?? []),
|
|
id
|
|
);
|
|
for (const warning of result.warnings ?? [])
|
|
recordProgress(id, { event: { kind: 'warning', message: warning } });
|
|
recordProgress(id, {
|
|
blocker: null,
|
|
event: {
|
|
kind: 'outcome',
|
|
message: result.ok
|
|
? state === 'warning'
|
|
? 'Completed with warnings'
|
|
: 'Completed'
|
|
: `Failed: ${result.message ?? 'Unknown error'}`,
|
|
details: {
|
|
state,
|
|
finishedAt,
|
|
error: result.message ?? null,
|
|
warnings: result.warnings ?? []
|
|
}
|
|
}
|
|
});
|
|
})();
|
|
}
|
|
|
|
/** Legacy failures also need a durable structured result before same-ID resume clears it. */
|
|
function preserveOutcome(id: string): void {
|
|
const row = db()
|
|
.prepare(
|
|
'SELECT state, finished_at, error, warnings, observation_json FROM server_deployments WHERE id = ?'
|
|
)
|
|
.get(id) as
|
|
| {
|
|
state: string;
|
|
finished_at: string | null;
|
|
error: string | null;
|
|
warnings: string;
|
|
observation_json: string | null;
|
|
}
|
|
| undefined;
|
|
if (!row || !['failed', 'interrupted', 'warning', 'done', 'cancelled'].includes(row.state))
|
|
return;
|
|
const previous = db()
|
|
.prepare(
|
|
"SELECT details_json FROM server_deployment_events WHERE deployment_id = ? AND kind = 'outcome' ORDER BY id DESC LIMIT 1"
|
|
)
|
|
.get(id) as { details_json: string | null } | undefined;
|
|
try {
|
|
const details = JSON.parse(previous?.details_json ?? 'null');
|
|
if (details?.state === row.state && details?.finishedAt === row.finished_at) return;
|
|
} catch {
|
|
/* Preserve the row's known outcome when old event metadata is missing. */
|
|
}
|
|
recordProgress(id, {
|
|
observedAt: readProgress(row)?.observedAt ?? null,
|
|
event: {
|
|
kind: 'outcome',
|
|
message: `${row.state === 'interrupted' ? 'Interrupted' : 'Recorded outcome'}: ${row.error ?? row.state}`,
|
|
details: {
|
|
state: row.state,
|
|
finishedAt: row.finished_at,
|
|
error: row.error,
|
|
warnings: warningMessages(row.warnings)
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
const activeDeployment =
|
|
"d.server_deleted_at IS NULL AND d.state IN ('queued', 'awaiting-health', 'running', 'waiting')";
|
|
const reviewDeployment = `d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS (
|
|
SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=d.id
|
|
AND c.resolved_at >= COALESCE(d.finished_at,d.started_at))`;
|
|
const deploymentListColumns = `d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage,
|
|
d.started_at, d.finished_at, d.triggered_by, d.requested_by, d.error, d.warnings,
|
|
d.phase_key AS phase, d.phase_started_at AS phaseStartedAt,
|
|
CASE WHEN json_valid(d.observation_json) THEN json_extract(d.observation_json,'$.blocker.code') END AS waitCode,
|
|
CASE WHEN json_valid(d.observation_json) THEN json_extract(d.observation_json,'$.blocker.message') END AS waitMessage,
|
|
(d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists,
|
|
(d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval,
|
|
(d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
|
|
AND ${rebalanceCancellationGuard}) AS canCancel
|
|
`;
|
|
|
|
function deploymentListQuery(options: ProvisioningListOptions) {
|
|
const conditions = [
|
|
options.view === 'active'
|
|
? `(${activeDeployment})`
|
|
: options.view === 'history'
|
|
? `NOT (${activeDeployment})`
|
|
: '1'
|
|
];
|
|
const parameters: string[] = [];
|
|
if (options.action) {
|
|
conditions.push('d.action = ?');
|
|
parameters.push(options.action);
|
|
}
|
|
if (options.status === 'attention') conditions.push(reviewDeployment);
|
|
else if (options.status === 'pending') conditions.push("d.state IN ('queued','waiting')");
|
|
else if (options.status) {
|
|
conditions.push('d.state=?');
|
|
parameters.push(options.status);
|
|
}
|
|
const linked = (
|
|
kinds: readonly string[]
|
|
) => `(EXISTS (SELECT 1 FROM provisioning_sources src WHERE src.operation_id=d.id AND src.kind IN (${kinds.map((kind) => `'${kind}'`).join(',')}))
|
|
OR ${kinds.map((kind) => `d.triggered_by LIKE '${kind}:%'`).join(' OR ')})`;
|
|
if (options.source === 'manual' || options.source === 'system') {
|
|
conditions.push(`NOT ${linked(Object.values(provisioningSourceKinds).flat())}`);
|
|
conditions.push(
|
|
options.source === 'manual' ? "d.triggered_by='ui'" : "d.triggered_by!='ui'"
|
|
);
|
|
} else if (options.source) conditions.push(linked(provisioningSourceKinds[options.source]));
|
|
const search = (options.server ?? '').slice(0, 100).toLowerCase();
|
|
if (search) {
|
|
// Read only distinct names to preserve friendly map/lobby searches, including deleted servers.
|
|
const names = (
|
|
db().prepare('SELECT DISTINCT server FROM server_deployments').all() as {
|
|
server: string;
|
|
}[]
|
|
)
|
|
.map((row) => row.server)
|
|
.filter((name) => `${name} ${serverTitle(name)}`.toLowerCase().includes(search));
|
|
conditions.push('d.server IN (SELECT value FROM json_each(?))');
|
|
parameters.push(JSON.stringify(names));
|
|
}
|
|
return {
|
|
where: conditions.join(' AND '),
|
|
parameters,
|
|
limit: provisioningPageSize(options.limit ?? 25),
|
|
offset: provisioningPageOffset(options.offset ?? 0)
|
|
};
|
|
}
|
|
|
|
function activeDeploymentRank(row: ProvisioningListRow): number {
|
|
if (row.state === 'awaiting-health') return 0;
|
|
if (row.state === 'running') return 1;
|
|
if (row.state === 'queued') return 3;
|
|
return 2;
|
|
}
|
|
|
|
function deploymentListRows(
|
|
query: ReturnType<typeof deploymentListQuery>,
|
|
active: boolean
|
|
): ProvisioningListRow[] {
|
|
const rows = db()
|
|
.prepare(
|
|
`SELECT ${deploymentListColumns} FROM server_deployments d WHERE ${query.where}
|
|
ORDER BY ${active ? 'd.started_at ASC, d.rowid ASC' : 'd.started_at DESC, d.rowid DESC LIMIT ? OFFSET ?'}`
|
|
)
|
|
.all(
|
|
REMOVING_SERVER,
|
|
...query.parameters,
|
|
...(active ? [] : [query.limit, query.offset])
|
|
) as ProvisioningListRow[];
|
|
if (!rows.length) return rows;
|
|
// Stable sorting keeps request/insertion order within each lifecycle state.
|
|
if (active)
|
|
rows.sort((left, right) => activeDeploymentRank(left) - activeDeploymentRank(right));
|
|
const links = db()
|
|
.prepare(
|
|
`SELECT operation_id,kind,source_id FROM provisioning_sources
|
|
WHERE operation_id IN (SELECT value FROM json_each(?)) ORDER BY kind,source_id`
|
|
)
|
|
.all(JSON.stringify(rows.map((row) => row.id))) as (Omit<
|
|
ProvisioningSource,
|
|
'label' | 'href'
|
|
> & { operation_id: string })[];
|
|
const byOperation = new Map<string, Omit<ProvisioningSource, 'label' | 'href'>[]>();
|
|
for (const { operation_id, ...link } of links)
|
|
byOperation.set(operation_id, [...(byOperation.get(operation_id) ?? []), link]);
|
|
const cache = new Map<string, ProvisioningSource>();
|
|
for (const row of rows) {
|
|
const sources = byOperation.get(row.id) ?? [];
|
|
const fallback = row.triggered_by.match(
|
|
/^(campaign|beta|maintenance|watchdog|autoscaling|rebalance):(.+)$/
|
|
);
|
|
if (
|
|
fallback &&
|
|
!sources.some(
|
|
(source) => source.kind === fallback[1] && source.source_id === fallback[2]
|
|
)
|
|
)
|
|
sources.unshift({
|
|
kind: fallback[1] as ProvisioningSource['kind'],
|
|
source_id: fallback[2]
|
|
});
|
|
row.sources = sources.map((source) => {
|
|
const key = `${source.kind}:${source.source_id}`;
|
|
if (!cache.has(key)) cache.set(key, sourceDetails(source));
|
|
return cache.get(key)!;
|
|
});
|
|
}
|
|
return rows;
|
|
}
|
|
|
|
/** All list reads omit observation snapshots; retained history is always bounded. */
|
|
export function listDeployments(options: ProvisioningListOptions = {}): ProvisioningListRow[] {
|
|
return deploymentListRows(deploymentListQuery(options), options.view === 'active');
|
|
}
|
|
|
|
export function deploymentListPage(options: ProvisioningListOptions = {}): ProvisioningListPage {
|
|
return db().transaction(() => {
|
|
const query = deploymentListQuery(options);
|
|
const deployments = deploymentListRows(query, options.view === 'active');
|
|
const total =
|
|
options.view === 'active'
|
|
? deployments.length
|
|
: (
|
|
db()
|
|
.prepare(
|
|
`SELECT COUNT(*) AS total FROM server_deployments d WHERE ${query.where}`
|
|
)
|
|
.get(...query.parameters) as { total: number }
|
|
).total;
|
|
const { needsReview } = db()
|
|
.prepare(
|
|
`SELECT COUNT(*) AS needsReview FROM server_deployments d WHERE ${reviewDeployment}`
|
|
)
|
|
.get() as { needsReview: number };
|
|
return { deployments, total, needsReview, limit: query.limit, offset: query.offset };
|
|
})();
|
|
}
|
|
|
|
export function interruptDeployments(): void {
|
|
db().transaction(() => {
|
|
const native =
|
|
'SELECT deployment_id FROM native_provisioning UNION ALL SELECT deployment_id FROM native_creation_intents';
|
|
const running = db()
|
|
.prepare(
|
|
`SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (${native})`
|
|
)
|
|
.all() as { id: string }[];
|
|
const finishedAt = new Date().toISOString();
|
|
const error =
|
|
'The application restarted before this operation finished. Check the server before retrying.';
|
|
db()
|
|
.prepare(
|
|
`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (${native})`
|
|
)
|
|
.run(finishedAt, error);
|
|
for (const { id } of running)
|
|
recordProgress(id, {
|
|
blocker: null,
|
|
event: {
|
|
kind: 'outcome',
|
|
message:
|
|
'Interrupted by application restart. Check the server before retrying.',
|
|
details: { state: 'interrupted', finishedAt, error }
|
|
}
|
|
});
|
|
})();
|
|
}
|