josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add provisioning lifecycle actions

Browse Source 
Rename Deployments to Provisioning with compatible redirects and APIs. Add durable Start, Restart, Stop, and Rebuild operations alongside Create and Destroy; rebuild wipes the instance and recreates it in the same slot and port.

Add lifecycle, migration, recovery, and Komodo command tests. Type checks and production build pass; the full suite retains a telemetry assertion failure reproduced on the unchanged branch.

Refs #39
This commit is contained in:
josh
2026-09-27 14:20:04 -04:00
parent 0b0e5b5bdb
commit cfed584614
42 changed files with 886 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/provisioning.ts
+17
View File
@@ -0,0 +1,17 @@
export const provisioningActions = ['create', 'start', 'restart', 'rebuild', 'stop', 'destroy'] as const;
export type ProvisioningAction = typeof provisioningActions[number];
export type RuntimeAction = Exclude<ProvisioningAction, 'create' | 'destroy'>;
export const actionLabels: Record<ProvisioningAction, string> = {
create: 'Create', start: 'Start', restart: 'Restart', rebuild: 'Rebuild', stop: 'Stop', destroy: 'Destroy'
};
export const actionCompleted: Record<ProvisioningAction, string> = {
create: 'Server creation completed', start: 'Server started', restart: 'Server restarted',
rebuild: 'Server rebuilt', stop: 'Server stopped', destroy: 'Server destroyed'
};
export const runtimeActions: RuntimeAction[] = ['start', 'restart', 'rebuild', 'stop'];
export const runtimeDescriptions: Record<RuntimeAction, string> = {
start: 'Start the existing container and verify its health.',
restart: 'Restart the container after the server is empty, then verify its health.',
rebuild: 'Destroy the instance and delete all of its server files, then create a fresh instance in the same server slot and port with the latest promoted setup. No instance data is retained. Uses the key saved separately in Keys. Waits until empty, then verifies health and map.',
stop: 'Stop the container after the server is empty. Its files, key, and registration are retained so you can start it again.'
};
src/lib/server/db/index.ts
+21
View File
@@ -227,6 +227,25 @@ function migrateGameTelemetry(handle: Database.Database): void {
})();
}
function migrateProvisioningActions(handle: Database.Database): void {
const table = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'").get() as { sql: string };
if (table.sql.includes("'rebuild'")) return;
// Rebuild the CHECK constraint without cascading into events, recipes, or
// collection members. Preserve rowids: they order same-server operations.
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(table.sql.replace('server_deployments', 'server_deployments_new')
.replace("'create', 'destroy'", "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"));
handle.exec(`INSERT INTO server_deployments_new (rowid, id, server, action, state, stage, started_at, finished_at, triggered_by, error, warnings)
SELECT rowid, id, server, action, state, stage, started_at, finished_at, triggered_by, error, warnings FROM server_deployments;
DROP TABLE server_deployments;
ALTER TABLE server_deployments_new RENAME TO server_deployments;`);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Provisioning migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
}
export function db(): Database.Database {
if (_db) return _db;
@@ -240,6 +259,8 @@ export function db(): Database.Database {
handle.pragma('foreign_keys = ON');
dropLegacyBetaTables(handle);
handle.exec(schemaSql);
migrateProvisioningActions(handle);
migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
migrateServers(handle);
// Column additions must run BEFORE any statement that references the
// new columns. schema.sql cannot include those references — see the
src/lib/server/db/schema.sql
+6 -1
View File
@@ -409,7 +409,7 @@ CREATE INDEX IF NOT EXISTS idx_telemetry_requests_received ON telemetry_requests
CREATE TABLE IF NOT EXISTS server_deployments (
id TEXT PRIMARY KEY,
server TEXT NOT NULL,
action TEXT NOT NULL CHECK (action IN ('create', 'destroy')),
action TEXT NOT NULL CHECK (action IN ('create', 'start', 'restart', 'rebuild', 'stop', 'destroy')),
state TEXT NOT NULL,
stage TEXT NOT NULL,
started_at TEXT NOT NULL,
@@ -436,6 +436,11 @@ CREATE TABLE IF NOT EXISTS server_creation_jobs (
expected_map TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS server_rebuild_jobs (
deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
removed_at TEXT
);
-- Blueprints retain map/count selections only. The legacy column name remains
-- compatible with older snapshot-based definitions, read as selections at runtime.
CREATE TABLE IF NOT EXISTS server_blueprints (
src/lib/server/fleet/collections.ts
+3 -3
View File
@@ -34,12 +34,12 @@ function coverage(entries: BlueprintEntry[]) {
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
}).map(server => server.name).sort();
const coverage = existingServers.map(name => {
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
: server?.container_state && !['running', 'starting', 'restarting'].includes(server.container_state) ? 'stopped'
: classifyHealth(server?.last_seen_at ?? null, server?.last_error ?? null, server?.container_state),
href: operation ? `/deployments/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
});
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
});
@@ -48,7 +48,7 @@ function coverage(entries: BlueprintEntry[]) {
function fleetSnapshot() {
return housePartyTargets().map(name => ({ name,
server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
}));
}
src/lib/server/fleet/create.ts
+77 -10
View File
@@ -42,7 +42,7 @@ export type CreateServerResult =
export type DestroyServerResult =
| { ok: true; warnings: string[] }
| { ok: false; kind: 'not-found' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
| { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
export interface CreateServerInput {
game?: ZombieGame; // omitted by older clients: T5
@@ -63,7 +63,7 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = []
const rows = db()
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
.all(`${baseName}_%`) as Array<{ name: string }>;
const taken = new Set<number>();
const prefix = `${baseName}_`;
@@ -87,7 +87,7 @@ export function nextAvailablePort(additionalPorts: number[] = [], releasingServe
const rows = db()
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
.all() as Array<{ name: string; game_port: number }>;
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
@@ -229,6 +229,63 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
})();
}
/** Reserve the same identity and port before removing any part of the instance. */
export function queueRebuildServer(name: string): CreateServerResult {
const server = getServer(name);
const game = gameOfServer(name);
const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
if (!server || !map || !server.game_port || server.komodo_type !== 'deployment' || server.komodo_name !== name
|| (server.host_dir && server.host_dir !== serverDir(name))) {
return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
}
if (!config.komodo.targetServer || !config.rcon.password) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
// Validate that the replacement setup can be composed before destruction.
const recipe = captureRecipe(game, map.slug);
return db().transaction(() => {
const id = beginDeployment(name, 'rebuild');
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
deploymentStage(id, 'Waiting to rebuild the empty server');
return { ok: true as const, name, port: server.game_port!, deploymentId: id };
})();
}
/** Caller has just verified emptiness and excluded conflicting work. */
export async function rebuildServer(id: string): Promise<void> {
const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
if (!job) return;
if (creating || destroying.size || creationQueue().active.length >= config.createConcurrency) {
// The cleanup checkpoint must survive capacity waits and app restarts.
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
return;
}
// Recheck prerequisites while the original instance still exists.
if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; }
try { captureRecipe(gameOfServer(job.server), job.map_slug); }
catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; }
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
destroying.add(job.server);
try {
const result = job.stage === REMOVING_SERVER
? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
if (!result.ok || result.warnings.length) {
finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
return;
}
db().transaction(() => {
db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
deploymentStage(id, 'Queued for server creation');
})();
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
finally { destroying.delete(job.server); }
}
function nextName(input: CreateServerInput): string | null {
const base = `${input.game ?? 't5'}_${input.gameMode}_${input.mapSlug}`;
const lobby = nextLobbyNumber(base);
@@ -437,6 +494,10 @@ function hasActiveCreation(name: string): boolean {
}
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action IN ('start','restart','stop','rebuild')
AND state IN ('waiting','queued','running')`).get(name)) {
return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
}
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
@@ -485,7 +546,11 @@ async function runDestructions(): Promise<void> {
continue;
}
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
const players = await rconGetPlayerCount(operation.server);
let players = await rconGetPlayerCount(operation.server);
if (players === null) {
const health = await komodoInspectHealthy(operation.server);
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
}
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
if (players !== 0) {
deploymentStage(operation.id, players === null
@@ -580,10 +645,12 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
};
}
const unfinished = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'awaiting-health'").all(name) as { id: string }[];
for (const { id } of unfinished) finishDeployment(id, { ok: false, message: 'Server destroyed before creation health checks completed' });
const unfinished = db().prepare("SELECT id, action FROM server_deployments WHERE server = ? AND state = 'awaiting-health'").all(name) as { id: string; action: string }[];
for (const { id, action } of unfinished) finishDeployment(id, { ok: false, message: action === 'create'
? 'Server destroyed before creation health checks completed' : 'Server destroyed before operation verification completed' });
return { ok: true, warnings };
}␍interface CreationJob {
}
interface CreationJob {
deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
}
@@ -636,7 +703,7 @@ export async function processCreations(): Promise<void> {
export function creationQueue() {
type Entry = { id: string; server: string; state: string; stage: string };
const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
WHERE action = 'create' AND state IN ('queued', 'running', 'awaiting-health')
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
ORDER BY started_at, rowid`).all() as Entry[];
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
}
@@ -656,7 +723,7 @@ async function runCreations(): Promise<CreationOutcomes> {
if (creating || destroying.size > 0) return outcomes;
// Restart recovery is observation-only: never repeat external creation or
// deployment. A known interrupted startup must retain its readiness gate.
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
for (const { id } of interrupted) {
if (resumeVerification(id)) continue;
// A queued removal does not make the interrupted container disappear.
@@ -717,7 +784,7 @@ async function runCreations(): Promise<CreationOutcomes> {
// API calls can cancel a selected job or resume an older startup while
// a previous provision awaits Komodo. Recheck without expanding this pass.
if (creationQueue().active.length >= config.createConcurrency) break;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action = 'create' AND state = 'queued'").get(id)) continue;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
const blocker = creationBlocker(id);
if (blocker) {
deploymentStage(id, blocker);
src/lib/server/fleet/deployments.ts
+21 -4
View File
@@ -1,12 +1,14 @@
import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { recipeForDeployment, summarizeRecipe } from './launch-recipes.js';
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
export function beginDeployment(server: string, action: 'create' | 'destroy', triggeredBy = 'ui'): string {
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
const id = randomUUID();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
recordDeploymentEvent(id, action === 'create' ? 'Server creation requested' : 'Server destruction requested');
recordDeploymentEvent(id, action === 'create' ? 'Server creation requested'
: action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested`);
return id;
}
@@ -32,7 +34,7 @@ export function getDeployment(id: string) {
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
return { deployment, events, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canCancelRemoval: canCancelRemoval(id), collection };
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection };
}
// This stage is written only after Komodo has confirmed deployment deletion.
@@ -77,11 +79,26 @@ export function canResumeVerification(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d
JOIN server_creation_jobs j ON j.deployment_id = d.id
JOIN servers s ON s.name = j.server AND s.game_port = j.port
WHERE d.id = ? AND d.action = 'create' AND d.state IN ('failed', 'interrupted')
WHERE d.id = ? AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id);
}
export function canResumeRebuild(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
WHERE d.id = ? AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id, REMOVING_SERVER);
}
export function resumeRebuild(id: string): boolean {
if (!canResumeRebuild(id)) return false;
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Resumed rebuild cleanup after confirmed Komodo deletion');
return true;
}
/** Resume observation only. Never repeat a container creation or restart. */
export function resumeVerification(id: string): boolean {
return db().transaction(() => {
src/lib/server/fleet/house-party.ts
+1 -1
View File
@@ -6,7 +6,7 @@ export function housePartyTargets(): string[] {
SELECT name FROM servers WHERE destroyed_at IS NULL
UNION
SELECT server AS name FROM server_deployments
WHERE action = 'create' AND state IN ('queued', 'running', 'awaiting-health')
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
ORDER BY name
`).all() as { name: string }[]).map(row => row.name);
}
src/lib/server/fleet/lifecycle.ts
+117
View File
@@ -0,0 +1,117 @@
import { db } from '../db/index.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { getServer } from './servers.js';
import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './deployments.js';
import { queueRebuildServer, rebuildServer } from './create.js';
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from '../pipeline/restart-rollout.js';
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
function blocker(server: string, id = ''): string | null {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
AND state IN ('queued','waiting','running','awaiting-health')`).get(server, id)) return 'Waiting for another provisioning operation on this server';
if (db().prepare(`SELECT 1 FROM deployment_servers WHERE server = ? AND restart_state IN ('pending','failed','held')
UNION ALL SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND b.state IN ('running','ready-promote')
UNION ALL SELECT 1 FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting') LIMIT 1`).get(server, server, server)) {
return 'Waiting for campaign, beta, or maintenance work on this server';
}
return rolloutWaitReason(server, id);
}
export function requestLifecycle(server: string, action: RuntimeAction) {
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
return db().transaction(() => {
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
if (existing) return existing.action === action
? { ok: true as const, deploymentId: existing.id }
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
const row = getServer(server);
if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
if (action === 'rebuild') {
const result = queueRebuildServer(server);
return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
: { ok: false as const, status: 400, error: result.message };
}
const id = beginDeployment(server, action);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
return { ok: true as const, deploymentId: id };
})();
}
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
let ticking: Promise<void> | null = null;
export function processLifecycle(): Promise<void> {
if (ticking) return ticking;
ticking = run().finally(() => { ticking = null; });
return ticking;
}
async function run(): Promise<void> {
await refreshRestartRollout();
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
ORDER BY started_at, rowid`).all() as Operation[];
for (const operation of operations) {
try { await processOperation(operation); }
catch (err) {
// Observation failures are retried; remote mutations are never replayed.
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message);
}
}
}
async function processOperation(operation: Operation): Promise<void> {
const { id, server, action } = operation;
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
if (!komodoConfigured() || !rconConfigured()) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
if (operation.state === 'awaiting-health') {
const health = await komodoInspectHealthy(server);
if (isStopped(health.state)) {
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
deploymentStage(id, 'Verified container stopped');
finishDeployment(id, { ok: true });
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
return;
}
const reason = blocker(server, id);
if (reason) { deploymentStage(id, reason); return; }
const health = await komodoInspectHealthy(server);
if (action !== 'start' && !isStopped(health.state)) {
const players = await rconGetPlayerCount(server);
if (players !== 0) {
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
return;
}
}
// Requests can arrive during the probes. Recheck before claiming the action.
if (blocker(server, id)) return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return;
if (action === 'rebuild') { await rebuildServer(id); return; }
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
if (!claim.changes) return;
if (action === 'start' || action === 'restart') {
const result = await restartInRollout(server, { operationId: id, action });
if (result.kind === 'queued') {
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, result.message);
}
return;
}
deploymentStage(id, 'Stopping container');
// Persist intent first; a lost reply must not issue another stop later.
try {
const result = await komodoRuntimeAction(server, 'stop');
if (!result.ok) { finishDeployment(id, { ok: false, message: result.message }); return; }
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for container to stop');
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
}
src/lib/server/fleet/overview.ts
+4 -2
View File
@@ -1,3 +1,4 @@
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
import { db } from '../db/index.js';
import type { FleetOperation } from '../../fleet-overview.js';
@@ -41,13 +42,14 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
.all(since) as { id: string; server: string; action: string; state: string; stage: string;
error: string | null; started_at: string }[];
for (const row of lifecycle) {
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
const state = ['done', 'warning', 'cancelled'].includes(row.state) ? 'done'
: ['failed', 'interrupted'].includes(row.state) ? 'attention' : 'active';
operations.push({ key: `deployment:${row.id}`, group: `deployment:${row.id}`, kind: 'deployment',
server: row.server, state, startedAt: row.started_at,
label: state === 'done' ? row.state === 'cancelled' ? 'Removal cancelled' : row.state === 'warning' ? 'Completed with warnings' : 'Complete'
: state === 'attention' ? `${row.action === 'create' ? 'Launch' : 'Removal'} needs attention` : row.stage,
detail: row.error ?? row.stage, href: `/deployments/${encodeURIComponent(row.id)}` });
: state === 'attention' ? `${actionLabels[row.action as ProvisioningAction]} needs attention` : row.stage,
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
}
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE b.state IN ('running', 'ready-promote')`)
src/lib/server/fleet/provisioning.ts
+1 -1
View File
@@ -5,5 +5,5 @@ export function isProvisioning(server: string): boolean {
const operation = db().prepare(`SELECT action, state FROM server_deployments
WHERE server = ? ORDER BY rowid DESC LIMIT 1`).get(server) as
{ action: string; state: string } | undefined;
return operation?.action === 'create' && ['queued', 'running', 'awaiting-health'].includes(operation.state);
return !!operation && ['create', 'rebuild', 'start'].includes(operation.action) && ['queued', 'running', 'awaiting-health'].includes(operation.state);
}
src/lib/server/integrations/komodo.ts
+15
View File
@@ -520,6 +520,21 @@ export async function komodoDeploy(deployment: string): Promise<KomodoWriteResul
return postAndClassify('execute/Deploy', { deployment });
}
/** Runtime controls use the saved Komodo resource, never the EC display name. */
export async function komodoRuntimeAction(server: string, action: 'start' | 'stop'): Promise<KomodoRestartResult> {
if (!komodoConfigured()) return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
const row = getServer(server);
if (!row?.komodo_name) return { ok: false, kind: 'no-name', message: 'no komodo_name set for server' };
const type = row.komodo_type ?? 'deployment';
const verb = action === 'start' ? 'Start' : 'Stop';
if (type === 'container') {
if (!row.komodo_server) return { ok: false, kind: 'no-name', message: 'container type needs komodo_server' };
return postAndClassify(`execute/${verb}Container`, { server: row.komodo_server, container: row.komodo_name });
}
if (type === 'stack') return postAndClassify(`execute/${verb}Stack`, { stack: row.komodo_name });
return postAndClassify(`execute/${verb}Deployment`, { deployment: row.komodo_name });
}
/** Stop + remove the container backing a Komodo deployment. Entity remains. */
export async function komodoDestroyDeployment(deployment: string): Promise<KomodoWriteResult> {
if (!komodoConfigured()) {
src/lib/server/pipeline/restart-rollout.ts
+32 -9
View File
@@ -1,7 +1,8 @@
import { config } from '../config.js';
import { db } from '../db/index.js';
import { komodoInspectHealthy, komodoRestart } from '../integrations/komodo.js';
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
import { rconStatusResult } from '../integrations/rcon.js';
import { beginDeployment, deploymentStage, finishDeployment } from '../fleet/deployments.js';
// Komodo queues restart work. Don't mistake the still-running old container
// for a completed restart when an API kick immediately follows the request.
@@ -15,6 +16,7 @@ interface RestartSlot {
healthy_at: string | null;
players: number | null;
error: string | null;
operation_id: string | null;
}
function slotFor(server: string): RestartSlot | undefined {
@@ -28,7 +30,12 @@ export function restartSettling(server: string): boolean {
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
}
export function rolloutWaitReason(server: string): string | null {
export function rolloutWaitReason(server: string, operationId = ''): string | null {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
return 'Waiting for the current provisioning operation';
}
const slot = slotFor(server);
if (slot?.healthy_at === null) {
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
@@ -43,14 +50,17 @@ export type RolloutRestartResult =
| { kind: 'failed'; message: string };
/** Reserve capacity before any async I/O; callers still check for players first. */
export async function restartInRollout(server: string): Promise<RolloutRestartResult> {
export async function restartInRollout(server: string, options: { operationId?: string; action?: 'start' | 'restart'; triggeredBy?: string } = {}): Promise<RolloutRestartResult> {
let operationId = options.operationId;
const waitReason = db().transaction(() => {
const reason = rolloutWaitReason(server);
const reason = rolloutWaitReason(server, operationId);
if (reason) return reason;
db().prepare(`INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)
operationId ??= beginDeployment(server, options.action ?? 'restart', options.triggeredBy ?? 'automation');
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`)
.run(server, new Date().toISOString());
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
.run(server, new Date().toISOString(), operationId);
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
return null;
}).immediate();
if (waitReason) return { kind: 'queued', message: waitReason };
@@ -59,8 +69,12 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
try {
const result = await komodoRestart(server);
if (result.ok) return { kind: 'started' };
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
if (result.ok) {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId);
deploymentStage(operationId!, 'Waiting for container and RCON health');
return { kind: 'started' };
}
// These failures cannot have queued a remote restart. Other failures
// (especially a timeout) may have: retain capacity until recovery.
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
@@ -68,10 +82,12 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
} else {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
}
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
return { kind: 'failed', message: `komodo: ${result.message}` };
} catch (err) {
const message = `komodo: ${(err as Error).message}`;
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
finishDeployment(operationId!, { ok: false, message });
return { kind: 'failed', message };
} finally {
// Start the grace period at completion, including slow/failed requests.
@@ -124,6 +140,13 @@ async function refresh(): Promise<void> {
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at);
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) {
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
deploymentStage(slot.operation_id, 'Verified container and RCON health');
finishDeployment(slot.operation_id, { ok: true });
}
}
}
}
src/lib/server/pipeline/worker.ts
+3
View File
@@ -1,4 +1,5 @@
import { processDestructions, processCreations } from '../fleet/create.js';
import { processLifecycle } from '../fleet/lifecycle.js';
import { sweep } from './sweep.js';
import { betaTick } from './beta-runner.js';
import { emit } from './bus.js';
@@ -37,6 +38,8 @@ async function runTick(): Promise<void> {
catch (err) { console.error('[worker] creation error:', err); }
try { await processDestructions(); }
catch (err) { console.error('[worker] destruction error:', err); }
try { await processLifecycle(); }
catch (err) { console.error('[worker] provisioning error:', err); }
try {
await sweep();
} catch (err) {
src/lib/ui/components/DeployCollectionDialog.svelte
+1 -1
View File
@@ -60,7 +60,7 @@
const body = await res.json();
if (!res.ok) throw new Error(body.error ?? 'Could not queue the collection.');
open = false;
if (body.collectionId) await goto(`/deployments/collections/${encodeURIComponent(body.collectionId)}`);
if (body.collectionId) await goto(`/provisioning/collections/${encodeURIComponent(body.collectionId)}`);
} catch (err) { error = (err as Error).message; }
finally { busy = false; }
}
src/lib/ui/components/DeployServerDialog.svelte
+3 -3
View File
@@ -73,7 +73,7 @@
const previousName = preview?.name;
busy = true; error = ''; savedName = ''; keyNotice = '';
try {
const res = await fetch('/api/deployments/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' },
const res = await fetch('/api/provisioning/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(sourceDeploymentId ? { deploymentId: sourceDeploymentId } : blueprintId ? { blueprintId } : { game, mapSlug }) });
const body = await res.json();
if (!res.ok) throw new Error(body.error ?? 'Could not prepare launch preview.');
@@ -97,7 +97,7 @@
const body = await res.json();
if (!res.ok || !body.ok) throw new Error(body.error ?? 'Could not queue this launch.');
open = false;
await goto(`/deployments/${encodeURIComponent(body.deploymentId)}`);
await goto(`/provisioning/${encodeURIComponent(body.deploymentId)}`);
} catch (err) { error = (err as Error).message; }
finally { busy = false; }
}
@@ -119,7 +119,7 @@
<dialog bind:this={dialog} aria-label="Create server" onclick={event => { if (isDialogBackdrop(event, dialog) && !busy && !saving) open = false; }} oncancel={event => { if (busy || saving) event.preventDefault(); }} onclose={() => open = false}>
<div class="composer">
<header><div><h2 bind:this={heading} tabindex="-1">{preview ? 'Review server launch' : 'Create server'}</h2></div><button class="close" disabled={busy || saving} onclick={() => open = false} aria-label="Close deployment composer">×</button></header>
<header><div><h2 bind:this={heading} tabindex="-1">{preview ? 'Review server launch' : 'Create server'}</h2></div><button class="close" disabled={busy || saving} onclick={() => open = false} aria-label="Close server creation dialog">×</button></header>
<ol class="steps" aria-label="Launch steps"><li class:active={!preview} aria-current={!preview ? 'step' : undefined}>01 <span>Choose a starting point</span></li><li class:active={!!preview} aria-current={preview ? 'step' : undefined}>02 <span>Review & launch</span></li></ol>
{#if error}<div class="error" role="alert">{error}</div>{/if}
{#if !loaded}
src/lib/ui/components/DestroyServerModal.svelte
+3 -3
View File
@@ -20,7 +20,7 @@
const res = await fetch(`/api/servers/${encodeURIComponent(name)}`, { method: 'DELETE' });
const body = await res.json();
if (!res.ok || !body.ok || !body.deploymentId) throw new Error(body.error ?? 'Could not queue destruction. Please retry.');
await goto(`/deployments/${encodeURIComponent(body.deploymentId)}`);
await goto(`/provisioning/${encodeURIComponent(body.deploymentId)}`);
open = false;
} catch (err) { error = (err as Error).message; }
finally { submitting = false; }
@@ -32,10 +32,10 @@
<form onsubmit={(event) => { event.preventDefault(); void destroy(); }}>
<h2 id="destroy-title">Queue server removal?</h2>
<p class="name">{name}</p>
<p>This stops the container, deletes its Komodo deployment, and removes its server files and campaign details. Deployment history and the saved server key are retained.</p>
<p>This stops the container, deletes its Komodo deployment, and removes its server files and campaign details. Provisioning history and the saved server key are retained.</p>
<div class="notice">
{#if players !== null && players > 0}<strong>{players} player{players === 1 ? '' : 's'} currently online.</strong>{/if}
Removal waits until a live check confirms the server is empty. Follow progress or cancel the waiting request on the deployment page.
Removal waits until a live check confirms the server is empty. Follow progress or cancel the waiting request on the provisioning page.
</div>
<p class="irreversible">You can cancel while waiting. Once removal begins, it cannot be undone.</p>
{#if error}<p role="alert" class="error">{error}</p>{/if}
src/lib/ui/components/HousePartyDialog.svelte
+3 -3
View File
@@ -18,7 +18,7 @@
async function review() {
busy = true; error = ''; servers = null;
try {
const res = await fetch('/api/deployments/house-party');
const res = await fetch('/api/provisioning/house-party');
if (!res.ok) throw new Error('Could not load the fleet.');
servers = (await res.json()).servers;
} catch (err) { error = (err as Error).message; }
@@ -29,7 +29,7 @@
if (busy || !servers?.length) return;
busy = true; error = '';
try {
const res = await fetch('/api/deployments/house-party', {
const res = await fetch('/api/provisioning/house-party', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ servers })
});
@@ -50,7 +50,7 @@
<header><h2>Queue fleet removal</h2></header>
<p>Queue destruction of every server, including servers being created. Each waits until a fresh player count confirms it is empty. Unreachable servers stay queued.</p>
<p>Destruction removes containers, server files, and fleet registrations. Saved keys and blueprints are kept.</p>
<p>In Active deployments, cancel individual waiting requests or all waiting removals. Once a server begins removal, that removal cannot be cancelled or undone.</p>
<p>In Provisioning’s Active tab, cancel individual waiting requests or all waiting removals. Once a server begins removal, that removal cannot be cancelled or undone.</p>
{#if error}<p class="error" role="alert">{error}</p>{/if}
{#if servers}
<strong>{servers.length ? `${servers.length} server${servers.length === 1 ? '' : 's'} to destroy` : 'The house is already empty.'}</strong>
src/lib/ui/components/LifecycleDialog.svelte
+70
View File
@@ -0,0 +1,70 @@
<script lang="ts">
import { goto } from '$app/navigation';
import { runtimeActions, actionLabels, runtimeDescriptions, type RuntimeAction } from '$lib/provisioning.js';
import { serverTitle } from '$ui/server-labels.js';
import Dialog from './primitives/Dialog.svelte';
import Button from './primitives/Button.svelte';
let { open = $bindable(false), initialServer = '', initialAction = 'start' }:
{ open?: boolean; initialServer?: string; initialAction?: RuntimeAction } = $props();
let servers = $state<{ name: string }[]>([]);
let server = $state('');
let action = $state<RuntimeAction | 'destroy'>('start');
let busy = $state(false);
let loading = $state(false);
let error = $state('');
let confirmed = $state(false);
const destructive = $derived(action === 'rebuild' || action === 'destroy');
$effect(() => {
if (open) { server = initialServer; action = initialAction; confirmed = false; error = ''; void load(); }
});
async function load() {
loading = true;
try {
const response = await fetch('/api/servers');
if (!response.ok) throw new Error('Could not load servers.');
servers = (await response.json()).servers;
} catch (err) { error = (err as Error).message; }
finally { loading = false; }
}
async function submit() {
if (busy || !server || (destructive && !confirmed)) return;
busy = true; error = '';
try {
const response = action === 'destroy'
? await fetch(`/api/servers/${encodeURIComponent(server)}`, { method: 'DELETE' })
: await fetch('/api/provisioning', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ server, action }) });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not queue this operation.');
await goto(`/provisioning/${encodeURIComponent(body.deploymentId)}`);
open = false;
} catch (err) { error = (err as Error).message; }
finally { busy = false; }
}
</script>
<Dialog bind:open title="Manage server" {busy}>
<div class="fields">
<label>Server <select bind:value={server} disabled={busy || loading} onchange={() => confirmed = false}>
<option value="">{loading ? 'Loading servers…' : 'Choose a server'}</option>
{#each servers as item}<option value={item.name}>{serverTitle(item.name)} · {item.name}</option>{/each}
</select></label>
<label>Action <select bind:value={action} disabled={busy} onchange={() => confirmed = false}>
{#each runtimeActions as value}<option value={value}>{actionLabels[value]}</option>{/each}
<option value="destroy">Destroy</option>
</select></label>
<p>{action === 'destroy' ? 'Destroy the container, delete its Komodo deployment, and remove all server files and registration after the server is empty. Provisioning history and the key saved separately in Keys are retained.' : runtimeDescriptions[action]}</p>
{#if destructive}<label class="confirmation"><input type="checkbox" bind:checked={confirmed} disabled={busy} /> I understand that all files and data in this instance will be permanently deleted.</label>{/if}
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}
</div>
{#snippet actions()}
<Button disabled={busy} onclick={() => open = false}>Cancel</Button>
<Button variant={destructive ? 'danger' : 'primary'} disabled={busy || loading || !server || (destructive && !confirmed)} loading={busy} onclick={submit}>Queue {actionLabels[action].toLowerCase()}</Button>
{/snippet}
</Dialog>
<style>
.fields { display: grid; gap: 16px; } label { display: grid; gap: 8px; }
select { padding: 10px; border: 1px solid var(--border-strong); border-radius: var(--radius-sm); background: var(--surface); color: var(--text-primary); }
p { margin: 0; line-height: 1.6; color: var(--text-secondary); } .confirmation { display: flex; align-items: flex-start; gap: 12px; line-height: 1.5; }
</style>
src/routes/+layout.svelte
+5 -5
View File
@@ -49,10 +49,10 @@
const stopServers = startServerStream();
const stopFleetMonitor = startFleetMonitor();
// Two-key nav shortcuts (Gmail-style): `g f` fleet, `g d` deployments,
// Two-key nav shortcuts (Gmail-style): `g f` fleet, `g d` provisioning,
// `g b` betas, `g s` scripts. Ignored while typing in inputs.
const shortcuts: Record<string, string> = {
f: '/', d: '/deployments', a: '/campaigns', b: '/betas',
f: '/', d: '/provisioning', a: '/campaigns', b: '/betas',
k: '/server-keys', s: '/scripts', c: '/configs', p: '/plugins', l: '/leaderboard', h: '/settings', m: '/metrics'
};
let armed = false;
@@ -99,7 +99,7 @@
const tabs = [
{ href: '/', label: 'Fleet', match: (p: string) => p === '/' || p.startsWith('/servers/') },
{ href: '/scripts', label: 'Library', match: (p: string) => /^\/(scripts|configs|plugins)(\/|$)/.test(p) },
{ href: '/deployments', label: 'Operations', match: (p: string) => /^\/(deployments|campaigns|server-keys)(\/|$)/.test(p) },
{ href: '/provisioning', label: 'Operations', match: (p: string) => /^\/(provisioning|deployments|campaigns|server-keys)(\/|$)/.test(p) },
{ href: '/betas', label: 'Betas', match: (p: string) => p.startsWith('/betas') },
{ href: '/settings', label: 'Settings', match: (p: string) => p.startsWith('/settings') },
{ href: '/leaderboard', label: 'Leaderboard', match: (p: string) => p.startsWith('/leaderboard') },
@@ -109,7 +109,7 @@
const secondary = $derived(tabs[1].match($page.url.pathname)
? [{ href: '/scripts', label: 'Scripts' }, { href: '/configs', label: 'Configs' }, { href: '/plugins', label: 'Plugins' }]
: tabs[2].match($page.url.pathname)
? [{ href: '/deployments', label: 'Deployments' }, { href: '/campaigns', label: 'Campaigns' }, { href: '/server-keys', label: 'Keys' }]
? [{ href: '/provisioning', label: 'Provisioning' }, { href: '/campaigns', label: 'Campaigns' }, { href: '/server-keys', label: 'Keys' }]
: tabs[4].match($page.url.pathname) ? [{ href: '/settings', label: 'Preferences' }] : []);
</script>
@@ -149,7 +149,7 @@
<Confirmation />
<NotificationHistory bind:open={notificationsOpen} />
<Dialog bind:open={helpOpen} title="Keyboard shortcuts" description="Press g, then the destination key. Shortcuts are inactive while typing or using a dialog.">
<dl class="shortcuts">{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Deployments'],['g a','Campaigns'],['g b','Betas'],['g h','Settings'],['g k','Keys'],['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}<div><dt><kbd>{keys}</kbd></dt><dd>{label}</dd></div>{/each}</dl>
<dl class="shortcuts">{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Provisioning'],['g a','Campaigns'],['g b','Betas'],['g h','Settings'],['g k','Keys'],['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}<div><dt><kbd>{keys}</kbd></dt><dd>{label}</dd></div>{/each}</dl>
</Dialog>
</div>
src/routes/+page.svelte
+1 -1
View File
@@ -173,7 +173,7 @@
{/if}
{#if servers.length === 0}
<EmptyState title="Your first server starts here" description="Check your connections and saved keys, then create a server or use a blueprint.">{#snippet actions()}<Button href="/settings">Check system setup</Button><Button href="/deployments?tab=blueprints" variant="ghost">Browse blueprints</Button>{/snippet}</EmptyState>
<EmptyState title="Your first server starts here" description="Check your connections and saved keys, then create a server or use a blueprint.">{#snippet actions()}<Button href="/settings">Check system setup</Button><Button href="/provisioning?view=blueprints" variant="ghost">Browse blueprints</Button>{/snippet}</EmptyState>
{:else}
<div class="toolbar">
<div class="search-row">
src/routes/api/deployments/[id]/resume/+server.ts
+2 -2
View File
@@ -1,9 +1,9 @@
import { json } from '@sveltejs/kit';
import { resumeVerification, resumeCleanup } from '$lib/server/fleet/deployments.js';
import { resumeVerification, resumeCleanup, resumeRebuild } from '$lib/server/fleet/deployments.js';
import { kick } from '$lib/server/pipeline/worker.js';
export function POST({ params }: { params: { id: string } }) {
if (!resumeVerification(params.id) && !resumeCleanup(params.id)) return json({ error: 'This operation cannot resume. Refresh and inspect its server.' }, { status: 409 });
if (!resumeVerification(params.id) && !resumeCleanup(params.id) && !resumeRebuild(params.id)) return json({ error: 'This operation cannot resume. Refresh and inspect its server.' }, { status: 409 });
void kick().catch(err => console.error('[deployment] verification kick failed:', err));
return json({ ok: true }, { status: 202 });
}
src/routes/api/provisioning/+server.ts
+20
View File
@@ -0,0 +1,20 @@
import { json } from '@sveltejs/kit';
import { listDeployments } from '$lib/server/fleet/deployments.js';
import { requestLifecycle } from '$lib/server/fleet/lifecycle.js';
import { runtimeActions, type RuntimeAction } from '$lib/provisioning.js';
import { kick } from '$lib/server/pipeline/worker.js';
export function GET() { return json({ deployments: listDeployments() }); }
export async function POST({ request }: { request: Request }) {
const body = await request.json().catch(() => null);
if (!body || typeof body.server !== 'string' || !body.server || !runtimeActions.includes(body.action)) {
return json({ error: 'server and a valid lifecycle action are required.' }, { status: 400 });
}
try {
const result = requestLifecycle(body.server, body.action as RuntimeAction);
if (!result.ok) return json({ error: result.error }, { status: result.status });
void kick().catch(err => console.error('[provisioning] worker kick failed:', err));
return json(result, { status: 202 });
} catch (err) { return json({ error: (err as Error).message }, { status: 400 }); }
}
src/routes/api/provisioning/[id]/+server.ts
+1
View File
@@ -0,0 +1 @@
export * from '../../deployments/[id]/+server.js';
src/routes/api/provisioning/[id]/cancel/+server.ts
+1
View File
@@ -0,0 +1 @@
export * from '../../../deployments/[id]/cancel/+server.js';
src/routes/api/provisioning/[id]/resume/+server.ts
+1
View File
@@ -0,0 +1 @@
export * from '../../../deployments/[id]/resume/+server.js';
src/routes/api/provisioning/house-party/+server.ts
+1
View File
@@ -0,0 +1 @@
export * from '../../deployments/house-party/+server.js';
src/routes/api/provisioning/preview/+server.ts
+1
View File
@@ -0,0 +1 @@
export * from '../../deployments/preview/+server.js';
src/routes/api/servers/[name]/+server.ts
+1 -1
View File
@@ -9,7 +9,7 @@ export async function DELETE({ params }: { params: { name: string } }) {
}
const result = await destroyServer(name);
if (!result.ok) {
const status = result.kind === 'not-found' ? 404 : 500;
const status = result.kind === 'not-found' ? 404 : result.kind === 'conflict' ? 409 : 500;
return json({ ok: false, error: result.message, kind: result.kind, warnings: result.warnings }, { status });
}
void kick().catch((err) => console.error('[destroy] worker kick failed:', err));
src/routes/api/system-status/+server.ts
+1 -1
View File
@@ -31,7 +31,7 @@ export async function GET({ url }: { url: URL }) {
{ id: 'rcon', title: 'Game server connection', state: !rconConfigured() ? 'unconfigured' : readyServers ? 'ready' : 'unverified', detail: !rconConfigured() ? 'Set RCON_PASSWORD to the password used by your game servers.' : readyServers ? `${readyServers} servers have fresh successful RCON reports.` : 'RCON password configured. A running server is needed to verify a response.', href: servers.length ? '/' : '#setup-guide', action: servers.length ? 'Review fleet' : 'Setup instructions' },
{ id: 'game-files', title: 'Game files on the host', state: 'unverified', detail: 'T5 and T6 game-file volumes must be populated on the Komodo host. Their contents cannot be verified from this app.', href: '#game-files', action: 'Game-file instructions' },
{ id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Key presence does not confirm validity with Plutonium.`, href: '/server-keys', action: savedKeys ? 'Manage keys' : 'Add a key' },
{ id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Servers are registered but none has a fresh healthy report. Open diagnostics to investigate.' : 'Create your first server after completing the setup checks.', href: servers.length ? '/' : '/deployments', action: servers.length ? 'Review fleet' : 'Create a server' }
{ id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Servers are registered but none has a fresh healthy report. Open diagnostics to investigate.' : 'Create your first server after completing the setup checks.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' }
];
const body: SystemStatus = { checkedAt: new Date().toISOString(), checks, readyServers, totalServers: servers.length, savedKeys };
return json(body, { headers: { 'cache-control': 'no-store' } });
src/routes/deployments/[...path]/+page.server.ts
+6
View File
@@ -0,0 +1,6 @@
import { redirect } from '@sveltejs/kit';
/** Preserve bookmarked operation links and list filters. */
export function load({ url }: { url: URL }) {
redirect(308, url.pathname.replace(/^\/deployments(?=\/|$)/, '/provisioning') + url.search);
}
src/routes/deployments/+page.svelte → src/routes/provisioning/+page.svelte
+21 -16
View File
@@ -20,8 +20,12 @@
import HousePartyDialog from '$ui/components/HousePartyDialog.svelte';
import type { BlueprintSummary, CollectionSummary } from '$lib/deployment-plan.js';
import LifecycleDialog from '$ui/components/LifecycleDialog.svelte';
import { provisioningActions, actionLabels, actionCompleted, type ProvisioningAction } from '$lib/provisioning.js';
let lifecycleOpen = $state(false);
interface Deployment {
id: string; server: string; action: 'create' | 'destroy';
id: string; server: string; action: ProvisioningAction;
state: 'queued' | 'awaiting-health' | 'waiting' | 'running' | 'done' | 'failed' | 'warning' | 'interrupted' | 'cancelled';
stage: string; started_at: string; finished_at: string | null;
triggered_by: string; error: string | null; warnings: string; server_exists: number;
@@ -51,7 +55,7 @@
$effect(() => {
const params = $page.url.searchParams;
tab = ['active', 'blueprints', 'history'].includes(params.get('view') ?? '') ? params.get('view')! : 'active';
action = ['create', 'destroy'].includes(params.get('action') ?? '') ? params.get('action')! : 'all';
action = provisioningActions.some(value => value === params.get('action')) ? params.get('action')! : 'all';
historyState = params.get('status') === 'attention' ? 'attention' : 'all';
search = params.get('server') ?? '';
limit = Math.max(25, Number(params.get('limit')) || 25);
@@ -68,7 +72,7 @@
async function cancelRequests(ids: string[]) {
cancelling = true; cancellationNotice = ''; let cancelled = 0; const unavailable: string[] = [];
for (const id of ids) {
try { const response = await fetch(`/api/deployments/${encodeURIComponent(id)}/cancel`, { method: 'POST' }); if (response.ok) cancelled++; else unavailable.push(id); }
try { const response = await fetch(`/api/provisioning/${encodeURIComponent(id)}/cancel`, { method: 'POST' }); if (response.ok) cancelled++; else unavailable.push(id); }
catch { unavailable.push(id); }
}
cancellationNotice = `${cancelled} queued removal${cancelled === 1 ? '' : 's'} cancelled.${cancelled ? ' The servers covered by those cancelled requests and their files were kept.' : ''}${unavailable.length ? ` ${unavailable.length} could not be cancelled; refresh their operations to see whether removal has begun.` : ''}`;
@@ -90,8 +94,8 @@
if (refreshing) return;
refreshing = true;
try {
const [res, collectionRes] = await Promise.all([fetch('/api/deployments'), fetch('/api/deployment-collections')]);
if (!res.ok || !collectionRes.ok) throw new Error('Could not load server deployments.');
const [res, collectionRes] = await Promise.all([fetch('/api/provisioning'), fetch('/api/deployment-collections')]);
if (!res.ok || !collectionRes.ok) throw new Error('Could not load provisioning activity.');
deployments = (await res.json()).deployments;
collections = (await collectionRes.json()).collections;
error = '';
@@ -126,12 +130,13 @@
function warnings(d: Deployment): string[] { try { return JSON.parse(d.warnings); } catch { return []; } }
</script>
<svelte:head><title>Deployments | Electric Cherry</title></svelte:head>
<svelte:head><title>Provisioning | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title="Deployments" subtitle="Create servers, apply blueprints, and track fleet changes.">
{#snippet actions()}<Button variant="primary" onclick={() => open = true}>Create server</Button><details class="fleet-actions"><summary>Fleet actions</summary><Button variant="danger" onclick={() => housePartyOpen = true}>Destroy fleet…</Button></details>{/snippet}
<SectionHeader title="Provisioning" subtitle="Create, start, restart, rebuild, stop, and destroy server instances.">
{#snippet actions()}<Button onclick={() => lifecycleOpen = true}>Manage server</Button><Button variant="primary" onclick={() => open = true}>Create server</Button><details class="fleet-actions"><summary>Fleet actions</summary><Button variant="danger" onclick={() => housePartyOpen = true}>Destroy fleet…</Button></details>{/snippet}
</SectionHeader>
<Tabs {tabs} bind:value={tab} onchange={() => saveContext(true)} ariaLabel="Deployment views" idPrefix="deployment-tabs" panelId="deployment-panel" />
<Tabs {tabs} bind:value={tab} onchange={() => saveContext(true)} ariaLabel="Provisioning views" idPrefix="deployment-tabs" panelId="deployment-panel" />
<LifecycleDialog bind:open={lifecycleOpen} />
<DeployServerDialog bind:open onblueprint={loadBlueprints} />
<BlueprintBuilder bind:open={builderOpen} blueprint={editingBlueprint} onsaved={blueprintSaved} />
<DeployCollectionDialog bind:open={collectionOpen} {blueprintId} />
@@ -174,7 +179,7 @@
<section>
<h2>Collection launches</h2>
<div class="rows">{#each visibleCollections.slice(0,collectionLimit) as collection (collection.id)}
<a class="deployment-row collection-row" href="/deployments/collections/{collection.id}">
<a class="deployment-row collection-row" href="/provisioning/collections/{collection.id}">
<div class="row-head"><strong>{collection.name}</strong><Badge variant={collection.active ? 'info' : collection.failed ? 'danger' : 'online'}>{collection.active ? collection.provisioning + collection.verifying ? collection.mode === 'exact' ? 'Applying blueprint' : 'Launching' : 'Queued' : collection.failed ? 'Needs attention' : 'Completed'}</Badge></div>
<div class="collection-counts"><span><strong>{collection.done} / {collection.total}</strong> {collection.mode === 'exact' ? 'completed' : 'ready'}</span>{#if collection.provisioning + collection.verifying}<span>{collection.provisioning + collection.verifying} {collection.mode === 'exact' ? 'in progress' : 'launching'}</span>{/if}{#if collection.queued}<span>{collection.queued} queued</span>{/if}{#if collection.failed}<span class="error">{collection.failed} need attention</span>{/if}</div>
<div class="collection-meter" aria-hidden="true"><span class="verified" style:width={`${collection.done / (collection.total || 1) * 100}%`}></span><span class="unsuccessful" style:width={`${collection.failed / (collection.total || 1) * 100}%`}></span></div>
@@ -189,15 +194,15 @@
{#if active.some(d => d.canCancelRemoval)}<div class="ui-summary"><span>Waiting removals can be withdrawn until removal begins. Running and completed removals cannot be reversed.</span><Button disabled={cancelling} onclick={() => cancelRequests(active.filter(d => d.canCancelRemoval).map(d => d.id))}>Cancel all waiting removals</Button></div>{/if}
<section>
<h2>In flight <span>{active.length}</span></h2>
{#if loading}<p>Loading deployments...</p>
{:else if active.length === 0}<EmptyState title="No server operations in flight" description="New creations and destructions appear here automatically." />
{#if loading}<p>Loading provisioning activity...</p>
{:else if active.length === 0}<EmptyState title="No server operations in flight" description="Server lifecycle operations appear here automatically." />
{:else}<div class="rows">{#each active as d (d.id)}{@render row(d)}{/each}</div>{/if}
</section>
{:else}
<section>
<h2>History</h2>
<div class="filters">
<label>Operation <select bind:value={action} onchange={event => { action = event.currentTarget.value; saveContext(true); }}><option value="all">All operations</option><option value="create">Creations</option><option value="destroy">Removals</option></select></label>
<label>Operation <select bind:value={action} onchange={event => { action = event.currentTarget.value; saveContext(true); }}><option value="all">All operations</option>{#each provisioningActions as value}<option value={value}>{actionLabels[value]}</option>{/each}</select></label>
<label>Status <select bind:value={historyState} onchange={event => { historyState = event.currentTarget.value; saveContext(true); }}><option value="all">All statuses</option><option value="attention">Needs review</option></select></label>
<label>Server <input bind:value={search} oninput={event => { search = event.currentTarget.value; saveContext(true); }} placeholder="Filter by server name" /></label>
</div>
@@ -211,14 +216,14 @@
</div>
{#snippet row(d: Deployment)}
<article class="deployment-row"><a class="row-link" href="/deployments/{d.id}">
<article class="deployment-row"><a class="row-link" href="/provisioning/{d.id}">
<div class="row-head">
<div><Badge variant={d.action === 'create' ? 'info' : 'neutral'}>{d.action === 'create' ? 'Creation' : 'Destruction'}</Badge>
<div><Badge variant={d.action === 'create' ? 'info' : 'neutral'}>{actionLabels[d.action]}</Badge>
<strong class="server"><ServerIdentity name={d.server} /></strong>
</div>
<StatusBadge state={d.state} />
</div>
<p>{d.state === 'done' ? (d.action === 'create' ? 'Server creation completed.' : 'Server destroyed.') : d.stage}</p>
<p>{d.state === 'done' ? actionCompleted[d.action] : d.stage}</p>
<div class="meta"><span>Started {fmt(d.started_at)}</span>{#if d.finished_at}<span>Finished {fmt(d.finished_at)}</span>{/if}<span>By {d.triggered_by}</span></div>
{#if d.error}<p class="error">{d.error}</p>{/if}
{#each warnings(d) as warning}<p class="error">{warning}</p>{/each}
src/routes/deployments/[id]/+page.server.ts → src/routes/provisioning/[id]/+page.server.ts
+1 -1
View File
@@ -7,5 +7,5 @@ export function load({ params }: { params: { id: string } }) {
if (db().prepare('SELECT id FROM deployments WHERE id = ?').get(params.id)) {
redirect(307, '/campaigns/' + encodeURIComponent(params.id));
}
error(404, 'Deployment not found');
error(404, 'Provisioning operation not found');
}
src/routes/deployments/[id]/+page.svelte → src/routes/provisioning/[id]/+page.svelte
+43 -24
View File
@@ -15,12 +15,14 @@
import LaunchRecipe from '$ui/components/LaunchRecipe.svelte';
import type { LaunchRecipeSummary } from '$lib/deployment-plan.js';
import { actionLabels, actionCompleted, type ProvisioningAction } from '$lib/provisioning.js';
interface Deployment {
id: string; expected_map?: string; server: string; action: 'create' | 'destroy'; state: string;
id: string; expected_map?: string; server: string; action: ProvisioningAction; state: string;
stage: string; started_at: string; finished_at: string | null;
triggered_by: string; error: string | null; warnings: string; server_exists: number;
}
interface Detail { deployment: Deployment; events: { id: number; created_at: string; message: string }[]; recipe: LaunchRecipeSummary | null; blueprintName: string | null; canResumeVerification: boolean; canResumeCleanup: boolean; canCancelRemoval: boolean; collection: { id: string; name: string } | null }
interface Detail { deployment: Deployment; events: { id: number; created_at: string; message: string }[]; recipe: LaunchRecipeSummary | null; blueprintName: string | null; canResumeVerification: boolean; canResumeCleanup: boolean; canResumeRebuild: boolean; canCancelRemoval: boolean; collection: { id: string; name: string } | null }
let detail = $state<Detail | null>(null);
let error = $state('');
let refreshing = false;
@@ -32,25 +34,40 @@
let warnings = $derived.by(() => {
try { return JSON.parse(deployment?.warnings ?? '[]') as string[]; } catch { return []; }
});
let steps = $derived(deployment?.action === 'destroy'
? ['Request destruction', 'Wait for empty server', 'Stop and remove container', 'Delete Komodo deployment', 'Remove server files and registry']
: ['Request creation', 'Prepare server files', 'Create Komodo deployment', 'Start container', 'Verify Komodo health', 'Verify RCON health', 'Verify selected map']);
const createSteps = ['Prepare server files', 'Create Komodo deployment', 'Start container', 'Verify Komodo health', 'Verify RCON health', 'Verify selected map'];
const createStages = ['Preparing server files', 'Creating Komodo deployment', 'Starting container', 'Waiting for Komodo health', 'Waiting for RCON health', 'Waiting for selected map'];
const destroySteps = ['Wait for empty server', 'Stop and remove container', 'Delete Komodo deployment', 'Remove server files and registry'];
const destroyStages = ['Waiting to verify', 'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'];
let steps = $derived(deployment?.action === 'destroy' ? ['Request destruction', ...destroySteps]
: deployment?.action === 'rebuild' ? ['Request rebuild', ...destroySteps, 'Queue fresh instance', ...createSteps]
: deployment?.action === 'create' ? ['Request creation', ...createSteps]
: ['Request operation', 'Wait for eligibility', `${deployment ? actionLabels[deployment.action] : 'Start'} container`, deployment?.action === 'stop' ? 'Verify container stopped' : 'Verify container and RCON health']);
let currentStep = $derived.by(() => {
if (!deployment) return 0;
const stages = deployment.action === 'destroy'
? ['Starting', 'Waiting', 'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry']
: ['Starting', 'Preparing server files', 'Creating Komodo deployment', 'Starting container', 'Waiting for Komodo health', 'Waiting for RCON health', 'Waiting for selected map'];
if (deployment.state === 'waiting' && deployment.stage !== 'Removing server files and registry') return 1;
if (deployment.state === 'awaiting-health' && deployment.stage.startsWith('Waiting for health verification')) return 4;
if (deployment.state === 'done' || deployment.state === 'warning') return stages.length;
return Math.max(0, stages.findIndex(stage => deployment.stage.startsWith(stage)));
if (deployment.state === 'done' || deployment.state === 'warning') return steps.length;
const { action, state, stage } = deployment;
if (state === 'waiting' && stage !== 'Removing server files and registry') return 1;
if (action === 'start' || action === 'restart' || action === 'stop') return state === 'awaiting-health' ? 3 : state === 'running' ? 2 : stage.includes('container') ? 2 : 1;
if (state === 'queued') return action === 'rebuild' ? 5 : 0;
const stages = action === 'destroy' ? ['Starting', ...destroyStages]
: action === 'rebuild' ? ['Starting', ...destroyStages, 'Queued for server creation', ...createStages]
: ['Starting', ...createStages];
const index = stages.findIndex(value => stage === value || stage.startsWith(value + ':'));
if (index < 0 && state === 'awaiting-health') return action === 'rebuild' ? 9 : 4;
return Math.max(0, index);
});
let explanation = $derived.by(() => {
if (!deployment) return '';
if (detail?.canResumeRebuild) return 'The old Komodo instance was deleted. Resume file cleanup, then create a fresh instance in the reserved server slot.';
if (['start', 'restart', 'stop'].includes(deployment.action)) {
if (deployment.state === 'done') return deployment.action === 'stop' ? 'Komodo confirmed the container is stopped. Start it again when ready.' : 'The container is healthy and RCON responds.';
if (['failed', 'interrupted'].includes(deployment.state)) return 'Inspect the server before submitting another operation. An uncertain response may have reached Komodo.';
return deployment.state === 'waiting' ? 'Waiting for other work and a safe server state before continuing.' : 'The request is saved. Verifying the resulting container state automatically.';
}
if (deployment.state === 'cancelled') return 'The queued removal was cancelled before removal began. The server and its files were kept.';
if (detail?.canResumeCleanup) return 'Komodo deletion succeeded. Resume the remaining file and database cleanup; the removed container does not need to answer RCON.';
if (deployment.state === 'waiting' && deployment.stage === 'Removing server files and registry') return 'Local cleanup is queued. Komodo deletion has already succeeded.';
if (deployment.state === 'done') return deployment.action === 'create'
if (deployment.state === 'done') return ['create', 'rebuild'].includes(deployment.action)
? 'The container is healthy, RCON responds, and the reported engine map matches your selection.'
: 'The container, deployment, and server registration have been removed.';
if (deployment.state === 'warning') return 'The operation finished with the warnings below. Review any remaining cleanup.';
@@ -58,11 +75,12 @@
? 'The server is still registered. Inspect its status, then resume health verification without creating or restarting a container.'
: 'Inspect the error and any remaining container or files before launching again. The activity below records how far this operation reached.';
if (deployment.state === 'queued') return 'Your request is saved in the launch queue. A startup slot opens after an earlier server passes its container, RCON, and map checks.';
if (deployment.action === 'rebuild' && deployment.state === 'waiting') return 'Waiting until the instance is empty and a creation slot is available. The old instance and all of its files will be deleted before a fresh instance is created.';
if (deployment.state === 'waiting') return 'The server stays online while players finish. Removal begins only after a live check confirms it is empty.';
if (deployment.stage.startsWith('Waiting for selected map')) return 'Container and RCON checks passed. Waiting for RCON to report the selected engine map. Inspect the server configuration if this persists.';
if (deployment.stage.startsWith('Waiting for RCON')) return 'The container is healthy. Waiting for the game to answer RCON; check its startup logs, password, and network if this persists.';
if (deployment.state === 'awaiting-health') return 'Waiting for the container to become healthy. Verification runs automatically; startup logs can help explain a delay.';
return deployment.action === 'create' ? 'Installing the latest promoted setup and starting its container. Health checks follow automatically.' : 'Removing the empty server and recording cleanup results.';
return ['create', 'rebuild'].includes(deployment.action) ? 'Installing the latest promoted setup and starting its container. Health checks follow automatically.' : 'Removing the empty server and recording cleanup results.';
});
function duration(start: string, end: number) {
const seconds = Math.max(0, Math.floor((end - new Date(start).getTime()) / 1000));
@@ -75,7 +93,7 @@
if (!deployment || resuming) return;
resuming = true;
try {
const res = await fetch(`/api/deployments/${encodeURIComponent(deployment.id)}/resume`, { method: 'POST' });
const res = await fetch(`/api/provisioning/${encodeURIComponent(deployment.id)}/resume`, { method: 'POST' });
const body = await res.json();
if (!res.ok) throw new Error(body.error ?? 'Could not resume the operation.');
await load();
@@ -86,7 +104,7 @@
if (!deployment || resuming) return;
resuming = true; error = '';
try {
const response = await fetch(`/api/deployments/${encodeURIComponent(deployment.id)}/cancel`, { method: 'POST' });
const response = await fetch(`/api/provisioning/${encodeURIComponent(deployment.id)}/cancel`, { method: 'POST' });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not cancel this request.');
await load();
@@ -98,8 +116,8 @@
refreshing = true;
const id = $page.params.id ?? '';
try {
const res = await fetch('/api/deployments/' + encodeURIComponent(id));
if (!res.ok) throw new Error(res.status === 404 ? 'Deployment not found.' : 'Could not refresh deployment.');
const res = await fetch('/api/provisioning/' + encodeURIComponent(id));
if (!res.ok) throw new Error(res.status === 404 ? 'Provisioning operation not found.' : 'Could not refresh provisioning operation.');
const body = await res.json();
if ($page.params.id === id) { detail = body; error = ''; }
} catch (err) { error = (err as Error).message; }
@@ -110,22 +128,23 @@
function fmt(iso: string) { return $timeFormat.dateTime(iso); }
</script>
<svelte:head><title>{deployment ? (deployment.action === 'destroy' ? 'Destroy ' : 'Create ') + deployment.server : 'Deployment'} | Electric Cherry</title></svelte:head>
<svelte:head><title>{deployment ? actionLabels[deployment.action] + ' ' + deployment.server : 'Provisioning'} | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title={deployment ? (deployment.action === 'destroy' ? 'Remove server' : 'Create server') : 'Deployment'} subtitle={deployment ? serverTitle(deployment.server) : undefined}>
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Deployments', href: '/deployments' }, { label: deployment ? serverTitle(deployment.server) : 'Details' }]} />{/snippet}
<SectionHeader title={deployment ? actionLabels[deployment.action] + ' server' : 'Provisioning'} subtitle={deployment ? serverTitle(deployment.server) : undefined}>
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Provisioning', href: '/provisioning' }, { label: deployment ? serverTitle(deployment.server) : 'Details' }]} />{/snippet}
</SectionHeader>
{#if error}<p role="alert" class="error">{error} <button onclick={load}>Retry</button></p>{/if}
{#if detail?.collection}<a href="/deployments/collections/{detail.collection.id}">← Back to {detail.collection.name} collection</a>{/if}
{#if detail?.collection}<a href="/provisioning/collections/{detail.collection.id}">← Back to {detail.collection.name} collection</a>{/if}
{#if deployment}
<section class="summary">
<div class="top"><StatusBadge state={deployment.state} />
{#if deployment.server_exists}<a href="/servers/{encodeURIComponent(deployment.server)}">View server</a>{/if}
</div>
<h2>{deployment.state === 'done' ? (deployment.action === 'destroy' ? 'Server destroyed' : 'Server creation completed') : deployment.stage}</h2>
<h2>{deployment.state === 'done' ? actionCompleted[deployment.action] : deployment.stage}</h2>
<p class="explanation" aria-live="polite">{explanation}</p>
<div class="timing"><span>Total time <strong>{duration(deployment.started_at, deployment.finished_at ? new Date(deployment.finished_at).getTime() : now)}</strong></span>{#if active && stageStarted}<span>This step <strong>{duration(stageStarted, now)}</strong></span>{/if}</div>
{#if detail?.canResumeVerification}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume health verification</Button></div>{/if}
{#if detail?.canResumeRebuild}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume rebuild cleanup</Button></div>{/if}
{#if detail?.canResumeCleanup}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume cleanup</Button></div>{/if}
{#if detail?.canCancelRemoval}<div><Button disabled={resuming} loading={resuming} onclick={cancelRemoval}>Cancel queued removal</Button><p class="muted">Available until removal begins. Cancelling keeps the server and its files.</p></div>{/if}
{#if deployment.server_exists && deployment.state !== 'done' && deployment.state !== 'cancelled'}<div class="recovery-links"><a href="/servers/{encodeURIComponent(deployment.server)}?tab=logs">Open startup logs</a><a href="/servers/{encodeURIComponent(deployment.server)}?tab=diagnostics">Diagnose connection</a>{#if deployment.stage.includes('map')}<a href="/servers/{encodeURIComponent(deployment.server)}?tab=config">Review map configuration</a>{/if}</div>{/if}
@@ -137,7 +156,7 @@
{#if detail?.recipe}<section class="saved-recipe"><div class="top"><h2>{detail.blueprintName ? `Launched from ${detail.blueprintName}` : 'Recorded launch files'}</h2><Button variant="ghost" onclick={() => open = true}>Launch another server →</Button></div><p class="muted">Launch this map with the latest promoted files using a fresh server ID and port. This does not repair the existing server.</p><LaunchRecipe recipe={detail.recipe} historical /></section><DeployServerDialog bind:open sourceDeploymentId={deployment.id} />{/if}
<OperationHistory steps={steps.map((label,index): OperationStep => ({label,state:deployment.state === 'cancelled' ? 'pending' : index < currentStep ? 'done' : index === currentStep ? (!active && ['failed','interrupted'].includes(deployment.state) ? 'failed' : 'current') : 'pending'}))} events={detail?.events ?? []} {active} />
{:else if !error}<p>Loading deployment...</p>{/if}
{:else if !error}<p>Loading provisioning operation...</p>{/if}
</div>
<style>
src/routes/deployments/collections/[id]/+page.server.ts → src/routes/provisioning/collections/[id]/+page.server.ts
View File
src/routes/deployments/collections/[id]/+page.svelte → src/routes/provisioning/collections/[id]/+page.svelte
+4 -4
View File
@@ -53,7 +53,7 @@
<svelte:head><title>{collection?.name ?? 'Collection launch'} | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title={collection?.name ?? 'Collection launch'}>
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Deployments', href: '/deployments' }, { label: 'Collection launch' }]} />{/snippet}
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Provisioning', href: '/provisioning' }, { label: 'Collection launch' }]} />{/snippet}
</SectionHeader>
{#if error}<p class="error" role="alert">{error} <button onclick={load}>Retry</button></p>{/if}
{#if collection && detail}
@@ -79,7 +79,7 @@
<section class="current-launches" aria-label="In progress">
<span class="eyebrow">In progress</span>
{#each currentServers as server (server.id)}
<a class="current-launch" href="/deployments/{server.id}">
<a class="current-launch" href="/provisioning/{server.id}">
<span class="activity" aria-hidden="true"></span>
<div><strong>{server.action === 'destroy' ? 'Removing: ' : ''}<ServerIdentity name={server.server} /></strong><p>{server.state === 'awaiting-health' ? 'Checking the container, connection, and expected map.' : server.stage}</p></div>
<span class="operation-link">Follow progress <span aria-hidden="true">→</span></span>
@@ -92,7 +92,7 @@
<div><strong>{waitingForPort(firstQueued) ? 'Waiting for a removal to release capacity' : otherLaunches.length ? 'Waiting for an earlier launch' : firstQueued?.queuePosition && firstQueued.queuePosition > 1 ? 'Earlier servers are ahead in the queue' : 'Your launch is up next'}</strong>
{#if otherLaunches.length}
<p>Your collection will start automatically as earlier launches finish. Currently launching:</p>
<div class="other-launches">{#each otherLaunches as active (active.id)}<a href="/deployments/{active.id}"><ServerIdentity name={active.server} /> <span aria-hidden="true">→</span></a>{/each}</div>
<div class="other-launches">{#each otherLaunches as active (active.id)}<a href="/provisioning/{active.id}"><ServerIdentity name={active.server} /> <span aria-hidden="true">→</span></a>{/each}</div>
{:else}<p>{firstQueued ? queueHint(firstQueued) : 'Your servers will begin automatically.'} Progress updates automatically.</p>{/if}
</div>
</aside>
@@ -108,7 +108,7 @@
<ol class="servers">
{#each detail.servers as server, index (server.id)}
<li class:launching={launching(server)} class:attention={needsAttention(server)}>
<a class="server" href="/deployments/{server.id}">
<a class="server" href="/provisioning/{server.id}">
<span class="order" class:ready={server.state === 'done'} aria-hidden="true">{server.state === 'done' ? '✓' : String(index + 1).padStart(2, '0')}</span>
<div class="server-content">
<div class="server-heading"><strong><ServerIdentity name={server.server} /></strong><Badge variant={needsAttention(server) ? 'danger' : server.state === 'done' ? 'online' : launching(server) ? 'info' : 'neutral'}>{stateLabel(server)}</Badge></div>
src/routes/servers/[name]/+page.svelte
+6 -1
View File
@@ -13,6 +13,10 @@
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import LifecycleDialog from '$ui/components/LifecycleDialog.svelte';
import { runtimeActions, actionLabels, type RuntimeAction } from '$lib/provisioning.js';
let lifecycleOpen = $state(false);
let lifecycleAction = $state<RuntimeAction>('start');
import DestroyServerModal from '$ui/components/DestroyServerModal.svelte';
import { telemetryByServer, uiNow } from '$ui/stores/servers.js';
import type { RosterPlayer, Health, RconEndpoint, RconFailure } from '$ui/stores/servers.js';
@@ -225,6 +229,7 @@
</script>
<svelte:head><title>{serverTitle(name)} · {activeTab.charAt(0).toUpperCase() + activeTab.slice(1)} · Electric Cherry</title></svelte:head>
<LifecycleDialog bind:open={lifecycleOpen} initialServer={name} initialAction={lifecycleAction} />
<DestroyServerModal bind:open={destroyOpen} {name} players={effectivePlayers} />
{#if server}
@@ -236,7 +241,7 @@
<Breadcrumbs items={[{ label: 'Fleet', href: '/' }, { label: serverTitle(name) }]} />
{/snippet}
{#snippet actions()}
<details class="server-actions"><summary>Server actions</summary><Button variant="danger" onclick={() => destroyOpen = true}>Queue server removal</Button></details>
<details class="server-actions"><summary>Server actions</summary>{#each runtimeActions as action}<Button onclick={() => { lifecycleAction = action; lifecycleOpen = true; }}>{actionLabels[action]}</Button>{/each}<Button variant="danger" onclick={() => destroyOpen = true}>Queue server removal</Button></details>
{/snippet}
</SectionHeader>