Clarify provisioning wait messages (fixes #54)
This commit is contained in:
@@ -4,6 +4,7 @@ import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../int
|
||||
import { probeServerStatus } from '../fleet/status-evidence.js';
|
||||
import { deploymentStage, finishDeployment } from './history.js';
|
||||
import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
|
||||
import type { RuntimeAction } from '../../provisioning.js';
|
||||
|
||||
// Komodo queues restart work. Don't mistake the still-running old container
|
||||
// for a completed restart when an API kick immediately follows the request.
|
||||
@@ -36,7 +37,7 @@ export function restartSettling(server: string): boolean {
|
||||
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
|
||||
}
|
||||
|
||||
export function rolloutWaitReason(server: string, operationId = ''): string | null {
|
||||
export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null {
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
|
||||
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
|
||||
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
|
||||
@@ -47,10 +48,11 @@ export function rolloutWaitReason(server: string, operationId = ''): string | nu
|
||||
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS (
|
||||
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId);
|
||||
if (slot?.healthy_at === null && !ownHandoff) {
|
||||
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
|
||||
return `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`;
|
||||
}
|
||||
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
|
||||
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
|
||||
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency
|
||||
? `Queued to ${action}; waiting for another server to recover` : null;
|
||||
}
|
||||
|
||||
export type RolloutRestartResult =
|
||||
@@ -63,7 +65,7 @@ export async function restartInRollout(server: string, options: { operationId: s
|
||||
const operationId = options.operationId;
|
||||
const waitReason = db().transaction(() => {
|
||||
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
|
||||
const reason = rolloutWaitReason(server, operationId);
|
||||
const reason = rolloutWaitReason(server, operationId, options.action);
|
||||
if (reason) return reason;
|
||||
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
||||
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
|
||||
|
||||
@@ -57,7 +57,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
|
||||
return held ? 'held' : ready ? 'ready' : 'cancelled';
|
||||
}
|
||||
|
||||
function blocker(server: string, id: string): string | null {
|
||||
function blocker(server: string, id: string, action: RuntimeAction): string | null {
|
||||
const sources = sourcesFor(id);
|
||||
const rank = priority(sources);
|
||||
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
|
||||
@@ -76,7 +76,7 @@ function blocker(server: string, id: string): string | null {
|
||||
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
|
||||
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
|
||||
LIMIT 1`).get(server, id)) return 'Waiting for campaign work on this server';
|
||||
return rolloutWaitReason(server, id);
|
||||
return rolloutWaitReason(server, id, action);
|
||||
}
|
||||
|
||||
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
|
||||
@@ -128,7 +128,7 @@ async function processOperation(operation: Operation): Promise<void> {
|
||||
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
|
||||
return;
|
||||
}
|
||||
const reason = blocker(server, id);
|
||||
const reason = blocker(server, id, action);
|
||||
if (reason) { deploymentStage(id, reason); return; }
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (action !== 'start' && !isStopped(health.state)) {
|
||||
@@ -140,7 +140,7 @@ async function processOperation(operation: Operation): Promise<void> {
|
||||
}
|
||||
}
|
||||
// Requests can arrive during the probes. Recheck before claiming the action.
|
||||
const latestReason = blocker(server, id);
|
||||
const latestReason = blocker(server, id, action);
|
||||
if (latestReason) { deploymentStage(id, latestReason); return; }
|
||||
if (sourceEligibility(operation) !== 'ready') return;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
|
||||
@@ -5,7 +5,7 @@ const states: Record<string, [string, StatusTone]> = {
|
||||
queued: ['Queued', 'neutral'], pending: ['Queued for restart', 'neutral'], running: ['In progress', 'info'], active: ['In progress', 'info'],
|
||||
deploying: ['Installing files', 'info'], restarting: ['Restarting', 'info'],
|
||||
observing: ['Collecting test evidence', 'info'], 'rolling-back': ['Rolling back', 'info'],
|
||||
'awaiting-empty': ['Waiting for empty lobby', 'neutral'], waiting: ['Waiting for empty lobby', 'neutral'],
|
||||
'awaiting-empty': ['Waiting for empty lobby', 'neutral'], waiting: ['Waiting', 'neutral'],
|
||||
'awaiting-health': ['Verifying health', 'info'], 'awaiting-match': ['Waiting for a match', 'neutral'],
|
||||
ready: ['Ready', 'online'], 'ready-promote': ['Ready to promote', 'online'],
|
||||
done: ['Completed', 'online'], passed: ['Test passed', 'online'], tested: ['Test passed', 'online'],
|
||||
|
||||
Reference in New Issue
Block a user