josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix server deletion cleanup and allow failed cleanup to resume

Browse Source
This commit is contained in:
josh
2026-09-18 00:49:30 -04:00
parent 274aedb66e
commit 453821805a
6 changed files with 148 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/create.ts
+31 -9
View File
@@ -1,5 +1,5 @@
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js';
import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER, resumeCleanup } from './deployments.js';
import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js';
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from './launch-recipes.js';
@@ -423,6 +423,8 @@ function hasActiveCreation(name: string): boolean {
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
const operation = db().transaction(() => {
const id = beginDeployment(name, 'destroy', triggeredBy);
@@ -441,11 +443,22 @@ export function processDestructions(): Promise<void> {
}
async function runDestructions(): Promise<void> {
const queued = db().prepare("SELECT id, server FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string }[];
const queued = db().prepare("SELECT id, server, stage FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string; stage: string }[];
for (const operation of queued) {
if (creating || destroying.has(operation.server)) continue;
destroying.add(operation.server);
try {
if (operation.stage === REMOVING_SERVER) {
const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
if (newer) {
finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
continue;
}
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(operation.id);
finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
continue;
}
if (!getServer(operation.server)) {
if (hasActiveCreation(operation.server)) {
deploymentStage(operation.id, 'Waiting for server creation to finish');
@@ -508,26 +521,35 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
}
// 3: host directory. Fall back to derived path if host_dir wasn't set on
return cleanupDestroyedServer(name, operation);
}
async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
const row = getServer(name);
const warnings: string[] = [];
// Persist the checkpoint before local work so failed cleanup can resume without RCON.
// Legacy operations already recorded this same stage after Komodo deletion.
deploymentStage(operation, REMOVING_SERVER);
// Host directory. Fall back to derived path if host_dir wasn't set on
// legacy rows.
deploymentStage(operation, 'Removing server files and registry');
const hostDir = row.host_dir ?? serverDir(name);
const hostDir = row?.host_dir ?? serverDir(name);
try {
await rm(hostDir, { recursive: true, force: true });
} catch (err) {
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
}
// 4: DB row + cascading rows. ON DELETE CASCADE covers server_scripts,
// beta_file_snapshots; the rest (deployment_servers, deployment_files,
// beta_servers, beta_server_scripts) reference servers(name) without
// cascade — clean them up explicitly.
// Remove server-owned beta and deployment rows before the registry row.
// Test evidence cascades from runs; inventory and snapshots cascade from servers.
try {
const tx = db().transaction((n: string) => {
db().prepare('DELETE FROM deployment_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM deployment_files WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_coverage WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_file_changes WHERE server = ?').run(n);
db().prepare('DELETE FROM servers WHERE name = ?').run(n);
});
tx(name);
src/lib/server/fleet/deployments.ts
+20 -1
View File
@@ -30,7 +30,26 @@ export function getDeployment(id: string) {
const collection = db().prepare(`SELECT c.id, c.name FROM server_collection_members m
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
return { deployment, events, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id), collection };
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), collection };
}
// This stage is written only after Komodo has confirmed deployment deletion.
export const REMOVING_SERVER = 'Removing server files and registry';
export function canResumeCleanup(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d
WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id, REMOVING_SERVER);
}
export function resumeCleanup(id: string): boolean {
return db().transaction(() => {
if (!canResumeCleanup(id)) return false;
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Resumed local cleanup after confirmed Komodo deletion');
return true;
})();
}
export function canResumeVerification(id: string): boolean {