josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Cancel active provisioning jobs when rolling back betas

Browse Source
This commit is contained in:
josh
2026-10-05 17:48:46 -04:00
parent 6ad5e7f4c6
commit 701066aaf8
25 changed files with 564 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/restarts.ts
+5
View File
@@ -1,5 +1,6 @@
import { fleetSettings } from '../settings.js';
import { db } from '../db/index.js';
import { provisioningCancelled } from './cancellation.js';
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
import { probeServerStatus } from '../fleet/status-evidence.js';
import { deploymentStage, finishDeployment } from './history.js';
@@ -45,6 +46,7 @@ export function rolloutWaitReason(server: string, operationId = '', action: Runt
}
export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
if (requestsInFlight.has(server)) return { code: 'operation-conflict', message: 'Waiting for the previous command submission to finish', automatic: true };
const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
@@ -74,6 +76,7 @@ export type RolloutRestartResult =
export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
const operationId = options.operationId;
const waitReason = db().transaction(() => {
if (provisioningCancelled(operationId)) return 'Operation was cancelled';
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
const reason = rolloutWaitReason(server, operationId, options.action);
if (reason) return reason;
@@ -111,6 +114,7 @@ export async function restartInRollout(server: string, options: { operationId: s
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
try {
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
if (result.ok) {
db().transaction(() => {
@@ -138,6 +142,7 @@ export async function restartInRollout(server: string, options: { operationId: s
})();
return { kind: 'failed', message: `komodo: ${result.message}` };
} catch (err) {
if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
const message = `komodo: ${(err as Error).message}`;
db().transaction(() => {