josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Recover unhealthy containers during create and rebuild

Browse Source
This commit is contained in:
josh
2026-10-08 13:08:33 -04:00
parent 012cd51a66
commit 368e457dc6
18 changed files with 881 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/index.ts
+14 -5
View File
@@ -466,10 +466,13 @@ export function listHostOperationHistory(page = 1, filter: HostOperationFilter =
/** Bound each cleanup pass so retained history cannot stall monitoring or receipts. */
export function pruneHostObservations(): number { return pruneHistory('observation'); }
/** Heartbeats can tell an executing create to stop at its next safe command boundary. */
/** Heartbeats stop either launch generation at its next safe command boundary. */
export function pendingCreationCancellations(hostId: string): string[] {
return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`).all(hostId) as { id: string }[]).map(row => row.id);
const rows = database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS creation_id,
json_extract(payload,'$.recoveryRebuildOperationId') AS recovery_id FROM ec_host_operations
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`)
.all(hostId) as { creation_id: string; recovery_id: string | null }[];
return rows.flatMap(row => row.recovery_id ? [row.creation_id, row.recovery_id] : [row.creation_id]);
}
type QueueHostOperationOptions = { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string };
export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: QueueHostOperationOptions = {}): HostOperation {
@@ -537,8 +540,11 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
// busy indefinitely. Drain the old batch by withholding ALL further work;
// heartbeats and receipts still flow, and the new capability releases this hold.
// This must live in the controller so the fix also works before an agent update.
if (!host.inventory.capabilities?.includes('instance.cancel-create') && database().prepare(`SELECT 1 FROM ec_host_operations
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
if (database().prepare(`SELECT 1 FROM ec_host_operations
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')
AND (NOT ? OR (json_extract(payload,'$.recoveryRebuildOperationId') IS NOT NULL AND NOT ?)) LIMIT 1`)
.get(hostId, Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false),
Number(host.inventory.capabilities?.includes('instance.unhealthy-recovery') ?? false))) return [];
// Accepted/running/unknown commands keep the same immutable ID. The agent
// consults its durable journal before acknowledging or executing a replay.
const healthOnly = `(kind IN ('instance.inspect','rcon.status-batch') OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
@@ -557,6 +563,9 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cancellation
JOIN native_provisioning creation ON creation.deployment_id=cancellation.creation_id
WHERE creation.operation_id=o.id)
${lane === 'health' ? "AND (state NOT IN ('accepted','running') OR MAX(updated_at,COALESCE(last_delivered_at,'')) <= ?)" : ''}
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 WHEN kind='plutonium.prepare' THEN 2 ELSE 1 END,
COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)