josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Roll deployment restarts through a shared fleet-wide recovery limit

Browse Source
This commit is contained in:
josh
2026-09-17 11:25:06 -04:00
parent 3f0d6dd87d
commit 8e04a190b3
12 changed files with 579 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/beta-runner.ts
+30 -18
View File
@@ -2,9 +2,9 @@ import { db } from '../db/index.js';
import { emit } from './bus.js';
import type { BetaStage } from './bus.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { scanForMatchEnd } from '../fleet/match-log.js';
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
interface BetaServerActiveRow {
beta_id: number;
@@ -84,8 +84,17 @@ function reconcileBetaState(betaId: number): void {
}
}
export async function betaTick(): Promise<{ processed: number }> {
let betaInFlight: Promise<{ processed: number }> | null = null;
export function betaTick(): Promise<{ processed: number }> {
if (betaInFlight) return betaInFlight;
betaInFlight = runBetaTick().finally(() => { betaInFlight = null; });
return betaInFlight;
}
async function runBetaTick(): Promise<{ processed: number }> {
if (!integrationsOn()) return { processed: 0 };
await refreshRestartRollout();
const rows = db()
.prepare(
@@ -95,7 +104,8 @@ export async function betaTick(): Promise<{ processed: number }> {
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
WHERE b.state = 'running'
AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')`
AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')
ORDER BY b.id, bs.server`
)
.all() as BetaServerActiveRow[];
@@ -117,19 +127,24 @@ export async function betaTick(): Promise<{ processed: number }> {
async function advance(row: BetaServerActiveRow): Promise<void> {
switch (row.stage) {
case 'awaiting-empty': {
case 'awaiting-empty':
case 'restarting': {
const waitReason = rolloutWaitReason(row.server);
if (waitReason) {
transition(row.beta_id, row.server, 'awaiting-empty', waitReason);
return;
}
const players = await rconGetPlayerCount(row.server);
if (players === null) return; // hold — RCON unreachable, try next tick
if (players > 0) return; // hold — still populated
transition(row.beta_id, row.server, 'restarting');
// Fall through to restart on the same tick for lower latency.
await advance({ ...row, stage: 'restarting' });
return;
}
case 'restarting': {
const r = await komodoRestart(row.server);
if (!r.ok) {
transition(row.beta_id, row.server, 'failed', `komodo: ${r.message}`);
const r = await restartInRollout(row.server);
if (r.kind === 'queued') {
transition(row.beta_id, row.server, 'awaiting-empty', r.message);
return;
}
if (r.kind === 'failed') {
transition(row.beta_id, row.server, 'failed', r.message);
return;
}
const next = row.wait_for_health ? 'awaiting-health' : row.wait_for_match ? 'awaiting-match' : 'ready';
@@ -137,14 +152,13 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
return;
}
case 'awaiting-health': {
const h = await komodoInspectHealthy(row.server);
const h = rolloutHealth(row.server);
if (!h.ok) {
// Record the failure reason (including any snippet) so the
// beta detail UI can surface why we're stuck.
const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
setStage().run(
'awaiting-health',
`unhealthy: ${h.reason}${snippet}`,
h.reason,
nowIso(),
'awaiting-health',
'awaiting-health',
@@ -159,8 +173,6 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
);
return;
}
const rcon = await rconStatusResult(row.server);
if (!rcon.ok) return; // container reports up but game not answering yet
const next = row.wait_for_match ? 'awaiting-match' : 'ready';
transition(row.beta_id, row.server, next);
return;