import { scalingAdmission } from '../autoscaling/admission.js'; import { maintenanceAllowed } from '../maintenance/policies.js'; import { db } from '../db/index.js'; import { getServer } from '../fleet/servers.js'; import { rolloutBlocker } from './restarts.js'; import { cancelProvisioning } from './requests.js'; import type { RuntimeAction } from '../../provisioning.js'; import type { ProvisioningBlocker } from '../../provisioning-detail.js'; import { processNativeProvisioning } from './native.js'; interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string; } const sourcesFor = (id: string) => db() .prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?') .all(id) as Source[]; const priority = (sources: Source[]) => sources.some((s) => s.kind === 'watchdog') ? 2 : sources.some((s) => s.kind === 'beta') ? 0 : sources.some((s) => s.kind === 'campaign') ? 1 : sources.some((s) => s.kind === 'maintenance') ? 3 : 2; /** Source work may hold or withdraw a request, but never executes it. */ function sourceEligibility( operation: Operation, onHold?: (source: Source) => void ): 'ready' | 'held' | 'cancelled' { const sources = sourcesFor(operation.id); if (!sources.length) return 'ready'; // Watchdog intent gates every linked source, including its drift campaign. for (const source of sources.filter((source) => source.kind === 'watchdog')) { if ( !db() .prepare( "SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')" ) .get(source.source_id, operation.server) || getServer(operation.server)?.desired_state === 'stopped' ) return 'cancelled'; const incident = db() .prepare('SELECT kind FROM watchdog_incidents WHERE id = ?') .get(source.source_id) as { kind: 'health' | 'drift' }; if (!maintenanceAllowed(incident.kind)) { onHold?.(source); return 'held'; } } let held = false; let ready = false; for (const source of sources) { if (source.kind === 'campaign') { const row = db() .prepare( 'SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL' ) .get(source.source_id, operation.server) as | { restart_state: string; files_changed: number; provisioning_id: string } | undefined; if (row?.provisioning_id !== operation.id || !row.files_changed) continue; if (['pending', 'failed'].includes(row.restart_state)) ready = true; if (['held', 'manual'].includes(row.restart_state)) { held = true; onHold?.(source); } } else if (source.kind === 'beta') { const row = db() .prepare( `SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL` ) .get(source.source_id, operation.server) as | { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined; if ( row?.provisioning_id !== operation.id || !['running', 'ready-promote'].includes(row.state) ) continue; if (!['awaiting-empty', 'restarting', 'awaiting-health'].includes(row.stage)) continue; if (!row.outcome || ['active', 'passed'].includes(row.outcome)) ready = true; if (row.outcome === 'paused') { held = true; onHold?.(source); } } else if (source.kind === 'maintenance') { const row = db() .prepare( 'SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL' ) .get(source.source_id) as | { state: string; provisioning_id: string; kind: string; origin: string } | undefined; if ( row?.provisioning_id === operation.id && ['queued', 'restarting'].includes(row.state) ) { if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled'; if ( !maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates')) ) { onHold?.(source); return 'held'; } ready = true; } } else ready = true; } return held ? 'held' : ready ? 'ready' : 'cancelled'; } function blocker(server: string, id: string, action: RuntimeAction): ProvisioningBlocker | null { const sources = sourcesFor(id); const rank = priority(sources); const others = db() .prepare( `SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')` ) .all(server, id) as { id: string; state: string; action: string; rowid: number }[]; const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number; }; const conflict = others.find( (other) => other.state !== 'waiting' || other.action === 'destroy' || priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid) ); if (conflict) { return { code: 'operation-conflict', message: 'Waiting for another provisioning operation on this server', automatic: true, href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' }; } // Beta activation must not wait on the campaign rows that the beta itself // holds. Campaign/maintenance callers likewise never block on their own job. const beta = rank > 0 ? (db() .prepare( `SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1` ) .get(server) as { id: string } | undefined) : undefined; if (beta) return { code: 'beta-work', message: 'Waiting for beta work on this server', automatic: true, href: `/betas/${encodeURIComponent(beta.id)}`, label: 'Blocking beta' }; const campaign = rank > 1 ? (db() .prepare( `SELECT ds.deployment_id FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS ( SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health'))) AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id) LIMIT 1` ) .get(server, id) as { deployment_id: string } | undefined) : undefined; if (campaign) return { code: 'campaign-work', message: 'Waiting for campaign work on this server', automatic: true, href: `/campaigns/${encodeURIComponent(campaign.deployment_id)}`, label: 'Blocking campaign' }; return rolloutBlocker(server, id, action); } interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string; } let awaitedPass: Promise | null = null; export function processProvisioning(options: { wait?: boolean } = {}): Promise { if (options.wait !== false && awaitedPass) return awaitedPass; // Each job owns its flight, so slow observations do not block new work. const pass = nativePass(options); if (options.wait === false) return pass; awaitedPass = pass.finally(() => { awaitedPass = null; }); return awaitedPass; } export const processLifecycle = processProvisioning; function nativePass(options: { wait?: boolean }): Promise { return processNativeProvisioning((operation) => { const scaling = scalingAdmission(operation.id); if (scaling) return scaling; if (operation.action === 'create') return null; if (operation.action === 'destroy') { const other = db() .prepare( `SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL AND (state='running' OR (state='awaiting-health' AND action NOT IN ('create','rebuild'))) LIMIT 1` ) .get(operation.server, operation.id); return other ? 'Waiting for the current server operation.' : null; } const typed = operation as Operation; const eligibility = sourceEligibility(typed); if (eligibility === 'cancelled') { cancelProvisioning(operation.id, 'Requester cancelled or replaced this operation'); return 'Requester cancelled this operation.'; } if (eligibility === 'held') return 'Waiting for the requester to resume.'; return blocker(operation.server, operation.id, typed.action)?.message ?? null; }, options); }