133 lines
4.7 KiB
TypeScript
133 lines
4.7 KiB
TypeScript
import { fleetSettings } from '../settings.js';
|
|
import { db } from '../db/index.js';
|
|
import type { RuntimeAction } from '../../provisioning.js';
|
|
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
|
|
|
const HEALTH_POLL_MS = 15_000;
|
|
|
|
interface RestartSlot {
|
|
server: string;
|
|
requested_at: string;
|
|
checked_at: string | null;
|
|
healthy_at: string | null;
|
|
players: number | null;
|
|
error: string | null;
|
|
operation_id: string | null;
|
|
}
|
|
|
|
function slotFor(server: string): RestartSlot | undefined {
|
|
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as
|
|
RestartSlot | undefined;
|
|
}
|
|
|
|
export function restartRecoveryPending(server: string): boolean {
|
|
return slotFor(server)?.healthy_at === null;
|
|
}
|
|
|
|
type SettlingSlot = Pick<RestartSlot, 'requested_at' | 'healthy_at' | 'operation_id'>;
|
|
interface SettlingOperation {
|
|
id: string;
|
|
action: string;
|
|
state: string;
|
|
}
|
|
|
|
/** Submitted lifecycle work stays Starting until verification ends, including slow restarts. */
|
|
export function recoverySettling(
|
|
slot: SettlingSlot | undefined,
|
|
operation?: SettlingOperation | null
|
|
): boolean {
|
|
if (!slot || slot.healthy_at) return false;
|
|
if (slot.operation_id)
|
|
return (
|
|
operation?.id === slot.operation_id &&
|
|
['start', 'restart', 'rebuild'].includes(operation.action) &&
|
|
['running', 'awaiting-health'].includes(operation.state)
|
|
);
|
|
// Legacy recovery slots have no operation whose verification can be followed.
|
|
return Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS;
|
|
}
|
|
|
|
export function restartSettling(server: string): boolean {
|
|
const slot = slotFor(server);
|
|
const operation = slot?.operation_id
|
|
? (db()
|
|
.prepare(
|
|
`SELECT id, action, state FROM server_deployments
|
|
WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`
|
|
)
|
|
.get(server) as SettlingOperation | undefined)
|
|
: undefined;
|
|
return recoverySettling(slot, operation);
|
|
}
|
|
|
|
export function rolloutWaitReason(
|
|
server: string,
|
|
operationId = '',
|
|
action: RuntimeAction = 'restart'
|
|
): string | null {
|
|
return rolloutBlocker(server, operationId, action)?.message ?? null;
|
|
}
|
|
|
|
export function rolloutBlocker(
|
|
server: string,
|
|
operationId = '',
|
|
action: RuntimeAction = 'restart'
|
|
): ProvisioningBlocker | null {
|
|
const conflict = db()
|
|
.prepare(
|
|
`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
|
|
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
|
|
OR (action = 'destroy' AND state IN ('waiting','running')))`
|
|
)
|
|
.get(server, operationId) as { id: string } | undefined;
|
|
if (conflict) {
|
|
return {
|
|
code: 'operation-conflict',
|
|
message: 'Waiting for the current provisioning operation',
|
|
automatic: true,
|
|
href: `/provisioning/${encodeURIComponent(conflict.id)}`,
|
|
label: 'Blocking operation'
|
|
};
|
|
}
|
|
const slot = slotFor(server);
|
|
const ownHandoff =
|
|
slot?.operation_id === operationId &&
|
|
!!db()
|
|
.prepare(
|
|
`SELECT 1 FROM server_deployments d
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS (
|
|
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`
|
|
)
|
|
.get(operationId);
|
|
if (slot?.healthy_at === null && !ownHandoff) {
|
|
return {
|
|
code: 'previous-recovery',
|
|
message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`,
|
|
automatic: true,
|
|
href: slot.operation_id
|
|
? `/provisioning/${encodeURIComponent(slot.operation_id)}`
|
|
: null,
|
|
label: 'Previous operation'
|
|
};
|
|
}
|
|
const { n } = db()
|
|
.prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL')
|
|
.get() as { n: number };
|
|
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >=
|
|
fleetSettings().restartConcurrency
|
|
? {
|
|
code: 'restart-capacity',
|
|
message: `Queued to ${action}; waiting for another server to recover`,
|
|
automatic: true
|
|
}
|
|
: null;
|
|
}
|
|
|
|
export function rolloutHealth(
|
|
server: string
|
|
): { ok: true; players: number } | { ok: false; reason: string } {
|
|
const slot = slotFor(server);
|
|
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
|
|
return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' };
|
|
}
|