56 lines
3.6 KiB
TypeScript
56 lines
3.6 KiB
TypeScript
import { fleetSettings } from '../settings.js';
|
|
import { db } from '../db/index.js';
|
|
import { emit } from './bus.js';
|
|
import { requestRestart } from '../provisioning/index.js';
|
|
import { pruneHostObservations } from '../hosts/index.js';
|
|
|
|
interface PendingRow {
|
|
deployment_id: string; server: string; restart_state: string;
|
|
restarted_at: string | null; files_changed: number;
|
|
players: number | null; error: string | null;
|
|
}
|
|
type SweepResult = { checked: number; restarted: number; healthy: number; held: number; failed: number };
|
|
let sweepInFlight: Promise<SweepResult> | null = null;
|
|
const nowIso = () => new Date().toISOString();
|
|
|
|
/** Submit activation requests and mirror their results. Provisioning owns all execution. */
|
|
export function sweep(): Promise<SweepResult> {
|
|
if (sweepInFlight) return sweepInFlight;
|
|
sweepInFlight = runSweep().finally(() => { sweepInFlight = null; });
|
|
return sweepInFlight;
|
|
}
|
|
async function runSweep(): Promise<SweepResult> {
|
|
pruneHostObservations();
|
|
const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
|
|
WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
|
|
const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
|
|
for (const row of pending) {
|
|
if (!row.files_changed) continue; // A failed file write is not a restart request.
|
|
const operation = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id });
|
|
result.checked++;
|
|
const state = operation.state === 'done' ? 'done'
|
|
: ['failed','interrupted','cancelled'].includes(operation.state) ? 'failed' : 'pending';
|
|
const restartedAt = ['awaiting-health','done'].includes(operation.state) ? operation.submitted_at : row.restarted_at;
|
|
if (restartedAt && !row.restarted_at) result.restarted++;
|
|
if (state === 'done') result.healthy++;
|
|
if (state === 'pending') result.held++;
|
|
if (state === 'failed') result.failed++;
|
|
const error = state === 'done' ? null : operation.error ?? operation.stage;
|
|
db().prepare(`UPDATE deployment_servers SET restart_state = ?, checked_at = ?, players = ?, error = ?,
|
|
attempts = attempts + CASE WHEN ? IS NOT NULL AND checked_at IS NOT ? THEN 1 ELSE 0 END,
|
|
restarted_at = COALESCE(restarted_at, ?) WHERE deployment_id = ? AND server = ?`)
|
|
.run(state, operation.checked_at, operation.players, error, operation.checked_at, operation.checked_at, restartedAt, row.deployment_id, row.server);
|
|
if (state !== row.restart_state || operation.players !== row.players || error !== row.error) {
|
|
emit({ type: 'server-state', deploymentId: row.deployment_id, server: row.server, state, players: operation.players, error });
|
|
}
|
|
}
|
|
db().prepare(`UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS (
|
|
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`).run(nowIso());
|
|
db().prepare(`DELETE FROM deployments WHERE started_at < ?
|
|
AND NOT EXISTS (SELECT 1 FROM watchdog_actions WHERE campaign_id = deployments.id) AND NOT EXISTS (
|
|
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`)
|
|
.run(new Date(Date.now() - fleetSettings().deployKeepDays * 86_400_000).toISOString());
|
|
emit({ type: 'sweep-tick', frozen: false });
|
|
return result;
|
|
}
|