josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Recover unhealthy containers during create and rebuild

Browse Source
This commit is contained in:
josh
2026-10-08 13:08:33 -04:00
parent 012cd51a66
commit 368e457dc6
18 changed files with 881 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/retention.ts
+17 -2
View File
@@ -22,12 +22,26 @@ const unsettledDeployment = `d.state NOT IN ${terminal}
(a.state NOT IN ('completed','failed') OR (a.id=n.operation_id AND a.state='completed' AND (
(n.applied_at IS NULL AND COALESCE(json_array_length(n.files_json),0)>0)
OR (d.action='destroy' AND i.retired_at IS NULL)
OR (d.action='rebuild' AND i.retired_at IS NULL AND i.generation=n.generation
AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id))))))
OR (a.kind='instance.rebuild' AND i.retired_at IS NULL AND i.generation=n.generation)))))
OR EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id
WHERE m.deployment_id=d.id AND (a.state NOT IN ('completed','failed') OR (a.state='completed' AND m.removed_at IS NULL)))`;
// Recovery replaces operation_id with its latest command. The original launch
// still supplies rebuild files and the ownership label used by creation cleanup.
const recoveryLaunch = `EXISTS (SELECT 1 FROM native_provisioning n
JOIN server_deployments d ON d.id=n.deployment_id
JOIN native_instances i ON i.id=n.instance_id AND i.retired_at IS NULL
WHERE n.recovery_step IS NOT NULL AND d.server_deleted_at IS NULL AND o.instance_id=i.id AND o.host_id=i.host_id
AND ((o.kind='instance.create' AND o.generation=n.generation)
OR (o.kind='instance.rebuild' AND o.generation+1=n.generation))
AND ((${unsettledDeployment}) OR (d.action='create' AND d.state IN ('failed','cancelled')
AND i.generation BETWEEN n.generation AND n.generation+1
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server
AND newer.server_deleted_at IS NULL AND newer.rowid>d.rowid AND newer.state!='cancelled'
AND NOT (newer.action='destroy' AND newer.state='waiting'
AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=newer.id AND p.operation_id IS NOT NULL))))))`;
const unreferencedOperation = `NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE o.id IN (n.operation_id,n.stage_operation_id))
AND NOT EXISTS (SELECT 1 FROM native_instances i WHERE i.retired_at IS NULL AND ${runtimeLaunchSql})
AND NOT (${recoveryLaunch})
AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.remove_operation_id=o.id)
AND NOT EXISTS (SELECT 1 FROM ec_host_decommissions d WHERE d.operation_id=o.id)
AND o.kind NOT IN ('runtime.pull','library.download','library.import','library.remove','host.decommission','host.rotate-credential')
@@ -97,6 +111,7 @@ function policies(): Policy[] {
failed: "o.state='failed'",
eligible: `o.state IN ('completed','failed') AND o.kind IN ('instance.create','instance.rebuild','instance.restart','instance.start','files.stage','files.read')
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='operation' AND c.record_id=o.id AND c.protected=1)
AND NOT (${recoveryLaunch})
AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE o.id IN (n.operation_id,n.stage_operation_id) AND ((${unsettledDeployment}) OR EXISTS (
SELECT 1 FROM retention_controls c WHERE c.category IN ('provisioning','workflow-payload') AND c.record_id=d.id AND c.protected=1)))