Fix server deletion cleanup and allow failed cleanup to resume

src/lib/server/fleet/deployments.ts

@@ -30,7 +30,26 @@ export function getDeployment(id: string) {
     const collection = db().prepare(`SELECT c.id, c.name FROM server_collection_members m
         JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
     return { deployment, events, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
-        canResumeVerification: canResumeVerification(id), collection };
+        canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), collection };
+}
+
+// This stage is written only after Komodo has confirmed deployment deletion.
+export const REMOVING_SERVER = 'Removing server files and registry';
+
+export function canResumeCleanup(id: string): boolean {
+    return !!db().prepare(`SELECT d.id FROM server_deployments d
+        WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
+        AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
+        .get(id, REMOVING_SERVER);
+}
+
+export function resumeCleanup(id: string): boolean {
+    return db().transaction(() => {
+        if (!canResumeCleanup(id)) return false;
+        db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
+        recordDeploymentEvent(id, 'Resumed local cleanup after confirmed Komodo deletion');
+        return true;
+    })();
 }
 
 export function canResumeVerification(id: string): boolean {