josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add provisioning lifecycle actions

Browse Source 
Rename Deployments to Provisioning with compatible redirects and APIs. Add durable Start, Restart, Stop, and Rebuild operations alongside Create and Destroy; rebuild wipes the instance and recreates it in the same slot and port.

Add lifecycle, migration, recovery, and Komodo command tests. Type checks and production build pass; the full suite retains a telemetry assertion failure reproduced on the unchanged branch.

Refs #39
This commit is contained in:
josh
2026-09-27 14:20:04 -04:00
parent 0b0e5b5bdb
commit cfed584614
42 changed files with 886 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/restart-rollout.ts
+32 -9
View File
@@ -1,7 +1,8 @@
import { config } from '../config.js';
import { db } from '../db/index.js';
import { komodoInspectHealthy, komodoRestart } from '../integrations/komodo.js';
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
import { rconStatusResult } from '../integrations/rcon.js';
import { beginDeployment, deploymentStage, finishDeployment } from '../fleet/deployments.js';
// Komodo queues restart work. Don't mistake the still-running old container
// for a completed restart when an API kick immediately follows the request.
@@ -15,6 +16,7 @@ interface RestartSlot {
healthy_at: string | null;
players: number | null;
error: string | null;
operation_id: string | null;
}
function slotFor(server: string): RestartSlot | undefined {
@@ -28,7 +30,12 @@ export function restartSettling(server: string): boolean {
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
}
export function rolloutWaitReason(server: string): string | null {
export function rolloutWaitReason(server: string, operationId = ''): string | null {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
return 'Waiting for the current provisioning operation';
}
const slot = slotFor(server);
if (slot?.healthy_at === null) {
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
@@ -43,14 +50,17 @@ export type RolloutRestartResult =
| { kind: 'failed'; message: string };
/** Reserve capacity before any async I/O; callers still check for players first. */
export async function restartInRollout(server: string): Promise<RolloutRestartResult> {
export async function restartInRollout(server: string, options: { operationId?: string; action?: 'start' | 'restart'; triggeredBy?: string } = {}): Promise<RolloutRestartResult> {
let operationId = options.operationId;
const waitReason = db().transaction(() => {
const reason = rolloutWaitReason(server);
const reason = rolloutWaitReason(server, operationId);
if (reason) return reason;
db().prepare(`INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)
operationId ??= beginDeployment(server, options.action ?? 'restart', options.triggeredBy ?? 'automation');
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`)
.run(server, new Date().toISOString());
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
.run(server, new Date().toISOString(), operationId);
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
return null;
}).immediate();
if (waitReason) return { kind: 'queued', message: waitReason };
@@ -59,8 +69,12 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
try {
const result = await komodoRestart(server);
if (result.ok) return { kind: 'started' };
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
if (result.ok) {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId);
deploymentStage(operationId!, 'Waiting for container and RCON health');
return { kind: 'started' };
}
// These failures cannot have queued a remote restart. Other failures
// (especially a timeout) may have: retain capacity until recovery.
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
@@ -68,10 +82,12 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
} else {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
}
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
return { kind: 'failed', message: `komodo: ${result.message}` };
} catch (err) {
const message = `komodo: ${(err as Error).message}`;
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
finishDeployment(operationId!, { ok: false, message });
return { kind: 'failed', message };
} finally {
// Start the grace period at completion, including slow/failed requests.
@@ -124,6 +140,13 @@ async function refresh(): Promise<void> {
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at);
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) {
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
deploymentStage(slot.operation_id, 'Verified container and RCON health');
finishDeployment(slot.operation_id, { ok: true });
}
}
}
}