josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Complete provisioning job detail pages and lifecycle evidence

Browse Source
This commit is contained in:
josh
2026-09-28 14:17:19 -04:00
parent 5b65520bbc
commit 3fb59cd598
41 changed files with 2866 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/requests.ts
+43 -12
View File
@@ -6,6 +6,7 @@ import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEve
import { queueRebuildServer } from './instances.js';
import { restartRequestInFlight } from './restarts.js';
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
import { recordProgress } from './progress.js';
export interface LifecycleOperation {
id: string; server: string; action: string; state: string; stage: string;
@@ -67,7 +68,8 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
if (!id) {
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty');
deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at);
}
db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)')
@@ -79,12 +81,17 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
/** Upgrade observation only: existing restart intent is never resubmitted. */
function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void {
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
{ operation_id: string | null; requested_at: string; healthy_at: string | null; players: number | null } | undefined;
const slot = db().prepare('SELECT operation_id, requested_at, checked_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
{ operation_id: string | null; requested_at: string; checked_at: string | null; healthy_at: string | null; players: number | null } | undefined;
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id);
recordProgress(id, { command: { state: 'attempted', at: submittedAt, message: 'Adopted a previously recorded submission; no command is replayed.' } });
if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) {
const finishedAt = new Date().toISOString();
const error = 'Existing restart cannot be correlated safely. Inspect the server before retrying.';
db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?")
.run(new Date().toISOString(), 'Existing restart cannot be correlated safely. Inspect the server before retrying.', id);
.run(finishedAt, error, id);
recordProgress(id, { recovery: { state: 'unknown', message: 'Existing restart recovery could not be correlated with this operation.', operationId: id },
observedAt: null, event: { kind: 'outcome', message: error, details: { state: 'interrupted', finishedAt, error } } });
return;
}
if (slot && slot.requested_at >= submittedAt) {
@@ -96,16 +103,24 @@ function adoptSubmittedRestart(id: string, server: string, submittedAt: string):
}
if (slot?.healthy_at && slot.requested_at >= submittedAt) {
db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id);
recordProgress(id, { observedAt: slot.healthy_at, event: { kind: 'outcome', message: 'Adopted recorded recovery completion.',
details: { state: 'done', finishedAt: slot.healthy_at, error: null, warnings: [] } } });
}
deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again');
deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again', {
phase: 'verify-container', checks: [], blocker: null, observedAt: slot?.checked_at ?? null,
recovery: { state: slot?.healthy_at && slot.requested_at >= submittedAt ? 'released' : 'held',
at: slot?.healthy_at && slot.requested_at >= submittedAt ? slot.healthy_at : undefined,
message: slot?.healthy_at && slot.requested_at >= submittedAt ? 'Adopted recorded recovery completion.' : 'Observing the previously submitted restart without replaying it.', operationId: id }
});
}
export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean {
return db().transaction(() => {
const finishedAt = new Date().toISOString();
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL
WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL`)
.run(new Date().toISOString(), reason, id);
if (result.changes) recordDeploymentEvent(id, reason);
.run(finishedAt, reason, id);
if (result.changes) recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt, error: null, warnings: [] } } });
return result.changes > 0;
})();
}
@@ -143,7 +158,8 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
const id = beginDeployment(server, action);
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
blocker: { code: 'eligibility', message: action === 'start' ? 'Waiting for start eligibility and recovery capacity.' : 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
return { ok: true as const, deploymentId: id };
})();
}
@@ -177,7 +193,8 @@ function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', inci
} else {
id = beginDeployment(server, action, `watchdog:${incidentId}`);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty');
deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
}
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server);
return { ok: true, deploymentId: id };
@@ -209,8 +226,14 @@ export function requestWatchdogLifecycle(server: string, action: 'restart' | 're
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
if (retained && !restartRequestInFlight(server)) {
db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(next.deploymentId, server, retained.operation_id);
if (transferred.changes) {
recordProgress(retained.operation_id, { recovery: { state: 'transferred', operationId: next.deploymentId,
message: 'Recovery capacity transferred to the explicit new incident attempt.' } });
recordProgress(next.deploymentId, { recovery: { state: 'held', operationId: next.deploymentId,
message: 'Recovery capacity retained from the prior incident attempt.' } });
}
recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId);
recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id);
}
@@ -249,8 +272,16 @@ export function escalateWatchdogRestart(incidentId: string, operationId: string,
const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId;
if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason });
else recordDeploymentEvent(operationId, reason);
if (slot) db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(replacement.deploymentId, reason, operation.server, operationId);
if (slot) {
const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(replacement.deploymentId, reason, operation.server, operationId);
if (transferred.changes) {
recordProgress(operationId, { recovery: { state: 'transferred', operationId: replacement.deploymentId,
message: 'Recovery capacity transferred to the rebuild operation.' } });
recordProgress(replacement.deploymentId, { recovery: { state: 'held', operationId: replacement.deploymentId,
message: 'Recovery capacity retained from the previous restart.' } });
}
}
recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server');
return replacement;
})();