Complete provisioning job detail pages and lifecycle evidence
This commit is contained in:
@@ -5,10 +5,12 @@ import { getServer } from '../fleet/servers.js';
|
||||
import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
|
||||
import { processCreations, processDestructions, rebuildServer } from './instances.js';
|
||||
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { probePlayerCount } from '../fleet/status-evidence.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from './restarts.js';
|
||||
import { probePlayerEvidence } from '../fleet/status-evidence.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
|
||||
import { cancelProvisioning } from './requests.js';
|
||||
import type { RuntimeAction } from '../../provisioning.js';
|
||||
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
|
||||
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
|
||||
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
|
||||
@@ -18,7 +20,7 @@ const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM pro
|
||||
const priority = (sources: Source[]) => sources.some(s => s.kind === 'watchdog') ? 2 : sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2;
|
||||
|
||||
/** Source work may hold or withdraw a request, but never executes it. */
|
||||
function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' {
|
||||
function sourceEligibility(operation: Operation, onHold?: (source: Source) => void): 'ready' | 'held' | 'cancelled' {
|
||||
const sources = sourcesFor(operation.id);
|
||||
if (!sources.length) return 'ready';
|
||||
// Watchdog intent gates every linked source, including its drift campaign.
|
||||
@@ -26,7 +28,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
|
||||
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(source.source_id, operation.server)
|
||||
|| getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
|
||||
const incident = db().prepare('SELECT kind FROM watchdog_incidents WHERE id = ?').get(source.source_id) as { kind: 'health' | 'drift' };
|
||||
if (!maintenanceAllowed(incident.kind)) return 'held';
|
||||
if (!maintenanceAllowed(incident.kind)) { onHold?.(source); return 'held'; }
|
||||
}
|
||||
let held = false;
|
||||
let ready = false;
|
||||
@@ -36,7 +38,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
|
||||
.get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined;
|
||||
if (row?.provisioning_id !== operation.id || !row.files_changed) continue;
|
||||
if (['pending','failed'].includes(row.restart_state)) ready = true;
|
||||
if (['held','manual'].includes(row.restart_state)) held = true;
|
||||
if (['held','manual'].includes(row.restart_state)) { held = true; onHold?.(source); }
|
||||
} else if (source.kind === 'beta') {
|
||||
const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`)
|
||||
@@ -44,12 +46,12 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
|
||||
if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue;
|
||||
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
|
||||
if (!row.outcome || ['active','passed'].includes(row.outcome)) ready = true;
|
||||
if (row.outcome === 'paused') held = true;
|
||||
if (row.outcome === 'paused') { held = true; onHold?.(source); }
|
||||
} else if (source.kind === 'maintenance') {
|
||||
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
|
||||
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) {
|
||||
if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
|
||||
if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) return 'held';
|
||||
if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) { onHold?.(source); return 'held'; }
|
||||
ready = true;
|
||||
}
|
||||
} else ready = true;
|
||||
@@ -57,26 +59,31 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
|
||||
return held ? 'held' : ready ? 'ready' : 'cancelled';
|
||||
}
|
||||
|
||||
function blocker(server: string, id: string, action: RuntimeAction): string | null {
|
||||
function blocker(server: string, id: string, action: RuntimeAction): ProvisioningBlocker | null {
|
||||
const sources = sourcesFor(id);
|
||||
const rank = priority(sources);
|
||||
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
|
||||
AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[];
|
||||
const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number };
|
||||
if (others.some(other => other.state !== 'waiting' || other.action === 'destroy'
|
||||
|| priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid))) {
|
||||
return 'Waiting for another provisioning operation on this server';
|
||||
const conflict = others.find(other => other.state !== 'waiting' || other.action === 'destroy'
|
||||
|| priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid));
|
||||
if (conflict) {
|
||||
return { code: 'operation-conflict', message: 'Waiting for another provisioning operation on this server', automatic: true,
|
||||
href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' };
|
||||
}
|
||||
// Beta activation must not wait on the campaign rows that the beta itself
|
||||
// holds. Campaign/maintenance callers likewise never block on their own job.
|
||||
if (rank > 0 && db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
|
||||
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
|
||||
const beta = rank > 0 ? db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server) as { id: string } | undefined : undefined;
|
||||
if (beta) return { code: 'beta-work', message: 'Waiting for beta work on this server', automatic: true, href: `/betas/${encodeURIComponent(beta.id)}`, label: 'Blocking beta' };
|
||||
const campaign = rank > 1 ? db().prepare(`SELECT ds.deployment_id FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
|
||||
AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
|
||||
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
|
||||
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
|
||||
LIMIT 1`).get(server, id)) return 'Waiting for campaign work on this server';
|
||||
return rolloutWaitReason(server, id, action);
|
||||
LIMIT 1`).get(server, id) as { deployment_id: string } | undefined : undefined;
|
||||
if (campaign) return { code: 'campaign-work', message: 'Waiting for campaign work on this server', automatic: true,
|
||||
href: `/campaigns/${encodeURIComponent(campaign.deployment_id)}`, label: 'Blocking campaign' };
|
||||
return rolloutBlocker(server, id, action);
|
||||
}
|
||||
|
||||
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
|
||||
@@ -100,7 +107,9 @@ async function run(): Promise<void> {
|
||||
try { await processOperation(operation); }
|
||||
catch (err) {
|
||||
// Observation failures are retried; remote mutations are never replayed.
|
||||
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message);
|
||||
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, {
|
||||
checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true }
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -112,36 +121,65 @@ async function processOperation(operation: Operation): Promise<void> {
|
||||
// Watchdog is paused; never strand the reserved replacement identity.
|
||||
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
|
||||
if (operation.state === 'waiting') {
|
||||
const eligibility = sourceEligibility(operation);
|
||||
const heldBy: Source[] = [];
|
||||
const eligibility = sourceEligibility(operation, source => heldBy.push(source));
|
||||
if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
|
||||
if (eligibility === 'held') { deploymentStage(id, 'Waiting for the requester to resume'); return; }
|
||||
if (eligibility === 'held') {
|
||||
const source = heldBy[0];
|
||||
const href = source ? source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}`
|
||||
: source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}`
|
||||
: source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}`
|
||||
: `/maintenance?view=activity#job-${encodeURIComponent(source.source_id)}` : null;
|
||||
deploymentStage(id, 'Waiting for the requester to resume', { phase: 'eligibility', checks: [],
|
||||
blocker: { code: 'requester-held', message: 'Waiting for the requester to resume', automatic: false, href, label: 'Requesting work' } });
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
|
||||
const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog');
|
||||
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
|
||||
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration', {
|
||||
phase: 'eligibility', checks: [], blocker: { code: 'configuration', message: 'Waiting for Komodo and RCON configuration', automatic: false, href: '/settings', label: 'Integration settings' }
|
||||
}); return; }
|
||||
if (operation.state === 'awaiting-health') {
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(id)) return;
|
||||
recordProgress(id, { phase: 'verify-stopped', checks: [{ key: 'stopped', label: 'Container stopped', source: 'komodo',
|
||||
result: isStopped(health.state) ? 'pass' : health.state ? 'waiting' : 'unknown', actual: health.state ?? null,
|
||||
message: health.ok ? null : health.reason }], blocker: isStopped(health.state) ? null
|
||||
: { code: 'stopped-state', message: 'Waiting for an explicit stopped container state.', automatic: true } });
|
||||
if (isStopped(health.state)) {
|
||||
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
|
||||
deploymentStage(id, 'Verified container stopped');
|
||||
finishDeployment(id, { ok: true });
|
||||
db().transaction(() => {
|
||||
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
|
||||
deploymentStage(id, 'Verified container stopped', { checkpoint: { key: 'verify-stopped', label: 'Stopped container state verified' } });
|
||||
finishDeployment(id, { ok: true });
|
||||
})();
|
||||
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
|
||||
return;
|
||||
}
|
||||
const reason = blocker(server, id, action);
|
||||
if (reason) { deploymentStage(id, reason); return; }
|
||||
if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (action !== 'start' && !isStopped(health.state)) {
|
||||
const players = await probePlayerCount(server);
|
||||
const evidence = await probePlayerEvidence(server);
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
const players = evidence.players;
|
||||
recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
|
||||
source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
|
||||
blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',
|
||||
message: players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`, automatic: true } });
|
||||
db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
|
||||
if (players !== 0) {
|
||||
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
|
||||
return;
|
||||
}
|
||||
} else if (action !== 'start') {
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
recordProgress(id, { phase: 'eligibility', blocker: null, checks: [{ key: 'stopped', label: 'Stopped container permits disruption',
|
||||
source: 'komodo', result: 'pass', actual: health.state ?? null }] });
|
||||
}
|
||||
// Requests can arrive during the probes. Recheck before claiming the action.
|
||||
const latestReason = blocker(server, id, action);
|
||||
if (latestReason) { deploymentStage(id, latestReason); return; }
|
||||
if (latestReason) { deploymentStage(id, latestReason.message, { phase: 'eligibility', blocker: latestReason }); return; }
|
||||
if (sourceEligibility(operation) !== 'ready') return;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
if (action === 'rebuild') { await rebuildServer(id); return; }
|
||||
@@ -155,13 +193,26 @@ async function processOperation(operation: Operation): Promise<void> {
|
||||
}
|
||||
return;
|
||||
}
|
||||
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
|
||||
deploymentStage(id, 'Stopping container');
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
|
||||
deploymentStage(id, 'Stopping container', { phase: 'container-action', blocker: null, checks: [],
|
||||
checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, command: { state: 'attempted', message: 'Stop submission attempted.' } });
|
||||
})();
|
||||
// Persist intent first; a lost reply must not issue another stop later.
|
||||
try {
|
||||
const result = await komodoRuntimeAction(server, 'stop');
|
||||
if (!result.ok) { finishDeployment(id, { ok: false, message: result.message }); return; }
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for container to stop');
|
||||
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
|
||||
if (!result.ok) {
|
||||
recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } });
|
||||
finishDeployment(id, { ok: false, message: result.message }); return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for container to stop', { phase: 'verify-stopped',
|
||||
checkpoint: { key: 'container-action', label: 'Stop command accepted' },
|
||||
command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } });
|
||||
})();
|
||||
} catch (err) {
|
||||
recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } });
|
||||
finishDeployment(id, { ok: false, message: (err as Error).message });
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user