Expire undelivered health checks when their wait times out

src/lib/server/hosts/index.ts

@@ -587,7 +587,25 @@ export async function waitHostOperationStatus(id: string, timeoutMs = 30_000): P
         const operation = getHostOperationStatus(id);
         if (!operation) throw new Error('Host operation was not found.');
         if (['completed','failed','unknown'].includes(operation.state)) return operation;
-        if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`);
+        if (Date.now() >= deadline) {
+            // Delivery and expiry both claim the pending row transactionally.
+            // Never cancel a delivered check or a command with side effects.
+            const expired = database().transaction(() => {
+                const timestamp = now();
+                const error = 'Expired before delivery.';
+                const result = database().prepare(`UPDATE ec_host_operations SET state='failed',error=?,updated_at=?
+                    WHERE id=? AND state='pending' AND last_delivered_at IS NULL
+                    AND kind IN ('instance.inspect','rcon.status-batch')`).run(error, timestamp, id);
+                if (result.changes) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)')
+                    .run(id, 'failed', timestamp, error);
+                return result.changes > 0;
+            })();
+            if (expired) {
+                queueMicrotask(() => workEvents.emit(`operation:${id}`));
+                throw new Error(`Host operation ${id} expired before delivery.`);
+            }
+            throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`);
+        }
         await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now())));
     }
 }