josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix server deletion cleanup and allow failed cleanup to resume

Browse Source
This commit is contained in:
josh
2026-09-18 00:49:30 -04:00
parent 274aedb66e
commit 453821805a
6 changed files with 148 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/create.ts
+31 -9
View File
@@ -1,5 +1,5 @@
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js';
import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER, resumeCleanup } from './deployments.js';
import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js';
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from './launch-recipes.js';
@@ -423,6 +423,8 @@ function hasActiveCreation(name: string): boolean {
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
const operation = db().transaction(() => {
const id = beginDeployment(name, 'destroy', triggeredBy);
@@ -441,11 +443,22 @@ export function processDestructions(): Promise<void> {
}
async function runDestructions(): Promise<void> {
const queued = db().prepare("SELECT id, server FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string }[];
const queued = db().prepare("SELECT id, server, stage FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string; stage: string }[];
for (const operation of queued) {
if (creating || destroying.has(operation.server)) continue;
destroying.add(operation.server);
try {
if (operation.stage === REMOVING_SERVER) {
const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
if (newer) {
finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
continue;
}
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(operation.id);
finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
continue;
}
if (!getServer(operation.server)) {
if (hasActiveCreation(operation.server)) {
deploymentStage(operation.id, 'Waiting for server creation to finish');
@@ -508,26 +521,35 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
}
// 3: host directory. Fall back to derived path if host_dir wasn't set on
return cleanupDestroyedServer(name, operation);
}
async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
const row = getServer(name);
const warnings: string[] = [];
// Persist the checkpoint before local work so failed cleanup can resume without RCON.
// Legacy operations already recorded this same stage after Komodo deletion.
deploymentStage(operation, REMOVING_SERVER);
// Host directory. Fall back to derived path if host_dir wasn't set on
// legacy rows.
deploymentStage(operation, 'Removing server files and registry');
const hostDir = row.host_dir ?? serverDir(name);
const hostDir = row?.host_dir ?? serverDir(name);
try {
await rm(hostDir, { recursive: true, force: true });
} catch (err) {
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
}
// 4: DB row + cascading rows. ON DELETE CASCADE covers server_scripts,
// beta_file_snapshots; the rest (deployment_servers, deployment_files,
// beta_servers, beta_server_scripts) reference servers(name) without
// cascade — clean them up explicitly.
// Remove server-owned beta and deployment rows before the registry row.
// Test evidence cascades from runs; inventory and snapshots cascade from servers.
try {
const tx = db().transaction((n: string) => {
db().prepare('DELETE FROM deployment_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM deployment_files WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_coverage WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_file_changes WHERE server = ?').run(n);
db().prepare('DELETE FROM servers WHERE name = ?').run(n);
});
tx(name);
src/lib/server/fleet/deployments.ts
+20 -1
View File
@@ -30,7 +30,26 @@ export function getDeployment(id: string) {
const collection = db().prepare(`SELECT c.id, c.name FROM server_collection_members m
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
return { deployment, events, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id), collection };
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), collection };
}
// This stage is written only after Komodo has confirmed deployment deletion.
export const REMOVING_SERVER = 'Removing server files and registry';
export function canResumeCleanup(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d
WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id, REMOVING_SERVER);
}
export function resumeCleanup(id: string): boolean {
return db().transaction(() => {
if (!canResumeCleanup(id)) return false;
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Resumed local cleanup after confirmed Komodo deletion');
return true;
})();
}
export function canResumeVerification(id: string): boolean {
src/routes/api/deployments/[id]/resume/+server.ts
+2 -2
View File
@@ -1,9 +1,9 @@
import { json } from '@sveltejs/kit';
import { resumeVerification } from '$lib/server/fleet/deployments.js';
import { resumeVerification, resumeCleanup } from '$lib/server/fleet/deployments.js';
import { kick } from '$lib/server/pipeline/worker.js';
export function POST({ params }: { params: { id: string } }) {
if (!resumeVerification(params.id)) return json({ error: 'This operation cannot resume verification. Refresh and inspect its server.' }, { status: 409 });
if (!resumeVerification(params.id) && !resumeCleanup(params.id)) return json({ error: 'This operation cannot resume. Refresh and inspect its server.' }, { status: 409 });
void kick().catch(err => console.error('[deployment] verification kick failed:', err));
return json({ ok: true }, { status: 202 });
}
src/routes/deployments/[id]/+page.svelte
+6 -3
View File
@@ -14,7 +14,7 @@
stage: string; started_at: string; finished_at: string | null;
triggered_by: string; error: string | null; warnings: string; server_exists: number;
}
interface Detail { deployment: Deployment; events: { id: number; created_at: string; message: string }[]; recipe: LaunchRecipeSummary | null; blueprintName: string | null; canResumeVerification: boolean; collection: { id: string; name: string } | null }
interface Detail { deployment: Deployment; events: { id: number; created_at: string; message: string }[]; recipe: LaunchRecipeSummary | null; blueprintName: string | null; canResumeVerification: boolean; canResumeCleanup: boolean; collection: { id: string; name: string } | null }
let detail = $state<Detail | null>(null);
let error = $state('');
let refreshing = false;
@@ -34,13 +34,15 @@
const stages = deployment.action === 'destroy'
? ['Starting', 'Waiting', 'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry']
: ['Starting', 'Preparing server files', 'Creating Komodo deployment', 'Starting container', 'Waiting for Komodo health', 'Waiting for RCON health', 'Waiting for selected map'];
if (deployment.state === 'waiting') return 1;
if (deployment.state === 'waiting' && deployment.stage !== 'Removing server files and registry') return 1;
if (deployment.state === 'awaiting-health' && deployment.stage.startsWith('Waiting for health verification')) return 4;
if (deployment.state === 'done' || deployment.state === 'warning') return stages.length;
return Math.max(0, stages.findIndex(stage => deployment.stage.startsWith(stage)));
});
let explanation = $derived.by(() => {
if (!deployment) return '';
if (detail?.canResumeCleanup) return 'Komodo deletion succeeded. Resume the remaining file and database cleanup; the removed container does not need to answer RCON.';
if (deployment.state === 'waiting' && deployment.stage === 'Removing server files and registry') return 'Local cleanup is queued. Komodo deletion has already succeeded.';
if (deployment.state === 'done') return deployment.action === 'create'
? 'The container is healthy, RCON responds, and the reported engine map matches your selection.'
: 'The container, deployment, and server registration have been removed.';
@@ -68,7 +70,7 @@
try {
const res = await fetch(`/api/deployments/${encodeURIComponent(deployment.id)}/resume`, { method: 'POST' });
const body = await res.json();
if (!res.ok) throw new Error(body.error ?? 'Could not resume verification.');
if (!res.ok) throw new Error(body.error ?? 'Could not resume the operation.');
await load();
} catch (err) { error = (err as Error).message; }
finally { resuming = false; }
@@ -112,6 +114,7 @@
<p class="explanation" aria-live="polite">{explanation}</p>
<div class="timing"><span>Total time <strong>{duration(deployment.started_at, deployment.finished_at ? new Date(deployment.finished_at).getTime() : now)}</strong></span>{#if active && stageStarted}<span>This step <strong>{duration(stageStarted, now)}</strong></span>{/if}</div>
{#if detail?.canResumeVerification}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume health verification</Button></div>{/if}
{#if detail?.canResumeCleanup}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume cleanup</Button></div>{/if}
{#if deployment.error}<p role="alert" class="error">{deployment.error}</p>{/if}
{#each warnings as warning}<p class="warning">{warning}</p>{/each}
{#if deployment.expected_map}<p>Expected map: <strong>{deployment.expected_map}</strong></p>{/if}