76 lines
4.2 KiB
TypeScript
76 lines
4.2 KiB
TypeScript
import { fleetSettings } from '../settings.js';
|
|
import { db } from '../db/index.js';
|
|
import type { RuntimeAction } from '../../provisioning.js';
|
|
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
|
|
|
const HEALTH_POLL_MS = 15_000;
|
|
|
|
interface RestartSlot {
|
|
server: string;
|
|
requested_at: string;
|
|
checked_at: string | null;
|
|
healthy_at: string | null;
|
|
players: number | null;
|
|
error: string | null;
|
|
operation_id: string | null;
|
|
}
|
|
|
|
function slotFor(server: string): RestartSlot | undefined {
|
|
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
|
|
}
|
|
|
|
export function restartRecoveryPending(server: string): boolean {
|
|
return slotFor(server)?.healthy_at === null;
|
|
}
|
|
|
|
type SettlingSlot = Pick<RestartSlot, 'requested_at' | 'healthy_at' | 'operation_id'>;
|
|
interface SettlingOperation { id: string; action: string; state: string }
|
|
|
|
/** Submitted lifecycle work stays Starting until verification ends, including slow restarts. */
|
|
export function recoverySettling(slot: SettlingSlot | undefined, operation?: SettlingOperation | null): boolean {
|
|
if (!slot || slot.healthy_at) return false;
|
|
if (slot.operation_id) return operation?.id === slot.operation_id
|
|
&& ['start', 'restart', 'rebuild'].includes(operation.action)
|
|
&& ['running', 'awaiting-health'].includes(operation.state);
|
|
// Legacy recovery slots have no operation whose verification can be followed.
|
|
return Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS;
|
|
}
|
|
|
|
export function restartSettling(server: string): boolean {
|
|
const slot = slotFor(server);
|
|
const operation = slot?.operation_id ? db().prepare(`SELECT id, action, state FROM server_deployments
|
|
WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as SettlingOperation | undefined : undefined;
|
|
return recoverySettling(slot, operation);
|
|
}
|
|
|
|
export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null {
|
|
return rolloutBlocker(server, operationId, action)?.message ?? null;
|
|
}
|
|
|
|
export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
|
|
const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
|
|
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
|
|
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
|
|
if (conflict) {
|
|
return { code: 'operation-conflict', message: 'Waiting for the current provisioning operation', automatic: true,
|
|
href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' };
|
|
}
|
|
const slot = slotFor(server);
|
|
const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS (
|
|
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId);
|
|
if (slot?.healthy_at === null && !ownHandoff) {
|
|
return { code: 'previous-recovery', message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`, automatic: true,
|
|
href: slot.operation_id ? `/provisioning/${encodeURIComponent(slot.operation_id)}` : null, label: 'Previous operation' };
|
|
}
|
|
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
|
|
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency
|
|
? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null;
|
|
}
|
|
|
|
export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
|
|
const slot = slotFor(server);
|
|
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
|
|
return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' };
|
|
}
|