Initial commit — SvelteKit fleet manager for T5 Plutonium servers
Ports push_scripts.sh (1497 lines of bash) to a SvelteKit + SQLite web app that manages a fleet of T5 Plutonium servers running on Komodo. Same push and beta workflow as the shell script, with a live SSE pipeline view, a !rt-token-gated auth modal that hard-pauses deployments on stale tokens, per-file audit diffs, and a Komodo-container deployment target. - Push/beta engines port the exact filter, held-by-beta, conflict, and stale-backup rules from the shell script - SQLite (better-sqlite3) replaces script_metadata.json; a one-shot importer moves legacy state over on first boot - 16 vitest tests cover push add/update/skip, beta open→rollback, open→promote, conflict guards, sweep frozen/manual branches, and IW4M login/status parsing - docker/ + komodo/stack.toml deploy as a managed container on the same host as the game servers, /root/zombies bind-mounted at the same path the shell script used
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
|
||||
// One in-process event bus, multiplexed by all three SSE endpoints. Kept
|
||||
// separate from any specific subscriber so push/beta/sweep can emit without
|
||||
// pulling the whole worker module in.
|
||||
|
||||
export type PipelineEvent =
|
||||
| { type: 'deployment-opened'; deploymentId: string; action: string; servers: string[] }
|
||||
| { type: 'server-changed'; deploymentId: string; server: string; files: number }
|
||||
| {
|
||||
type: 'server-state';
|
||||
deploymentId: string;
|
||||
server: string;
|
||||
state: 'pending' | 'done' | 'failed' | 'held' | 'manual';
|
||||
players: number | null;
|
||||
error: string | null;
|
||||
}
|
||||
| { type: 'sweep-tick'; frozen: boolean; reason?: string }
|
||||
| { type: 'auth-state'; state: import('../integrations/auth-state.js').AuthState; msRemaining?: number }
|
||||
| { type: 'server-status'; entries: Array<{ server: string; players: number | null; name?: string }> };
|
||||
|
||||
class Bus extends EventEmitter {
|
||||
constructor() {
|
||||
super();
|
||||
// Deployments and sweep can produce a lot of events during a big push.
|
||||
this.setMaxListeners(64);
|
||||
}
|
||||
}
|
||||
|
||||
const bus = new Bus();
|
||||
|
||||
export function emit(event: PipelineEvent): void {
|
||||
bus.emit('event', event);
|
||||
}
|
||||
|
||||
export function subscribe(listener: (event: PipelineEvent) => void): () => void {
|
||||
bus.on('event', listener);
|
||||
return () => bus.off('event', listener);
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import { config, iw4mConfigured, komodoConfigured } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { emit } from './bus.js';
|
||||
import { fetchStatus, playersOn } from '../integrations/iw4m.js';
|
||||
import { komodoRestart } from '../integrations/komodo.js';
|
||||
import { pipelineReady } from '../integrations/auth-state.js';
|
||||
|
||||
interface PendingRow {
|
||||
deployment_id: string;
|
||||
server: string;
|
||||
attempts: number;
|
||||
}
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function integrationsOn(): boolean {
|
||||
return komodoConfigured() && iw4mConfigured();
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-check every pending/failed deployment_server row, restart the ones with
|
||||
* zero players, mark the busy ones held. Same-server dedup is per-sweep, so
|
||||
* one restart resolves every pending row for that server. Ports `deploy_sweep`
|
||||
* in push_scripts.sh:705.
|
||||
*/
|
||||
export async function sweep(): Promise<{ checked: number; restarted: number; held: number; failed: number; frozen: boolean }> {
|
||||
if (!pipelineReady()) {
|
||||
emit({ type: 'sweep-tick', frozen: true, reason: 'iw4m token stale' });
|
||||
return { checked: 0, restarted: 0, held: 0, failed: 0, frozen: true };
|
||||
}
|
||||
|
||||
if (!integrationsOn()) {
|
||||
// Same behavior as line 714 — mark rows manual so they don't hang
|
||||
// forever if integrations were never configured.
|
||||
db()
|
||||
.prepare(
|
||||
`UPDATE deployment_servers
|
||||
SET restart_state = 'manual', checked_at = ?, error = 'integrations not configured'
|
||||
WHERE restart_state IN ('pending','failed')`
|
||||
)
|
||||
.run(nowIso());
|
||||
return { checked: 0, restarted: 0, held: 0, failed: 0, frozen: false };
|
||||
}
|
||||
|
||||
const pending = db()
|
||||
.prepare(
|
||||
`SELECT deployment_id, server, attempts
|
||||
FROM deployment_servers
|
||||
WHERE restart_state IN ('pending','failed')`
|
||||
)
|
||||
.all() as PendingRow[];
|
||||
if (pending.length === 0) return { checked: 0, restarted: 0, held: 0, failed: 0, frozen: false };
|
||||
|
||||
// Warm the /api/status cache so per-server calls just hit memory
|
||||
const status = await fetchStatus();
|
||||
if (!status.ok) {
|
||||
// Same as line 742 — mark held with the iw4m error
|
||||
const stmt = db().prepare(
|
||||
`UPDATE deployment_servers
|
||||
SET checked_at = ?, error = ?, attempts = attempts + 1
|
||||
WHERE restart_state IN ('pending','failed')`
|
||||
);
|
||||
stmt.run(nowIso(), `iw4m: ${status.message}`);
|
||||
return { checked: 0, restarted: 0, held: pending.length, failed: 0, frozen: false };
|
||||
}
|
||||
|
||||
// Dedup across deployments: if we resolved server X once this tick,
|
||||
// reuse the outcome for every other pending row on X.
|
||||
type Outcome = { state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null };
|
||||
const seen = new Map<string, Outcome>();
|
||||
|
||||
let checked = 0;
|
||||
let restarted = 0;
|
||||
let held = 0;
|
||||
let failed = 0;
|
||||
|
||||
const applyOutcome = db().prepare(
|
||||
`UPDATE deployment_servers
|
||||
SET restart_state = ?, checked_at = ?, players = ?, error = ?, attempts = attempts + 1,
|
||||
restarted_at = CASE WHEN ? = 'done' THEN ? ELSE restarted_at END
|
||||
WHERE deployment_id = ? AND server = ?`
|
||||
);
|
||||
|
||||
for (const row of pending) {
|
||||
let outcome = seen.get(row.server);
|
||||
if (!outcome) {
|
||||
checked++;
|
||||
const players = await playersOn(row.server);
|
||||
if (players === null) {
|
||||
outcome = { state: 'pending', players: null, error: 'no iw4m match' };
|
||||
held++;
|
||||
} else if (players > 0) {
|
||||
outcome = { state: 'pending', players, error: null };
|
||||
held++;
|
||||
} else {
|
||||
const r = await komodoRestart(row.server);
|
||||
if (r.ok) {
|
||||
outcome = { state: 'done', players: 0, error: null };
|
||||
restarted++;
|
||||
} else if (r.kind === 'no-name') {
|
||||
outcome = { state: 'failed', players: 0, error: r.message };
|
||||
failed++;
|
||||
} else {
|
||||
outcome = { state: 'failed', players: 0, error: `komodo: ${r.message}` };
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
seen.set(row.server, outcome);
|
||||
}
|
||||
const now = nowIso();
|
||||
applyOutcome.run(
|
||||
outcome.state,
|
||||
now,
|
||||
outcome.players,
|
||||
outcome.error,
|
||||
outcome.state,
|
||||
now,
|
||||
row.deployment_id,
|
||||
row.server
|
||||
);
|
||||
emit({
|
||||
type: 'server-state',
|
||||
deploymentId: row.deployment_id,
|
||||
server: row.server,
|
||||
state: outcome.state,
|
||||
players: outcome.players,
|
||||
error: outcome.error
|
||||
});
|
||||
}
|
||||
|
||||
// Finish deployments where every server is now non-pending
|
||||
db()
|
||||
.prepare(
|
||||
`UPDATE deployments
|
||||
SET finished_at = ?
|
||||
WHERE finished_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM deployment_servers
|
||||
WHERE deployment_id = deployments.id
|
||||
AND restart_state IN ('pending','failed')
|
||||
)`
|
||||
)
|
||||
.run(nowIso());
|
||||
|
||||
// Prune rows older than DEPLOY_KEEP_DAYS that have no pending/failed servers
|
||||
const cutoff = new Date(Date.now() - config.deployKeepDays * 24 * 60 * 60 * 1000).toISOString();
|
||||
db()
|
||||
.prepare(
|
||||
`DELETE FROM deployments
|
||||
WHERE started_at < ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM deployment_servers
|
||||
WHERE deployment_id = deployments.id
|
||||
AND restart_state IN ('pending','failed')
|
||||
)`
|
||||
)
|
||||
.run(cutoff);
|
||||
|
||||
emit({ type: 'sweep-tick', frozen: false });
|
||||
return { checked, restarted, held, failed, frozen: false };
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { sweep } from './sweep.js';
|
||||
import { emit } from './bus.js';
|
||||
import { fetchStatus } from '../integrations/iw4m.js';
|
||||
import { listServers, seedServerRegistry } from '../fleet/servers.js';
|
||||
import { getAuthState } from '../integrations/auth-state.js';
|
||||
|
||||
const TICK_MS = 15_000;
|
||||
const STATUS_REFRESH_MS = 30_000;
|
||||
|
||||
let started = false;
|
||||
let tickTimer: NodeJS.Timeout | null = null;
|
||||
let statusTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
async function tick(): Promise<void> {
|
||||
try {
|
||||
await sweep();
|
||||
} catch (err) {
|
||||
console.error('[worker] sweep error:', err);
|
||||
}
|
||||
}
|
||||
|
||||
async function pushServerStatus(): Promise<void> {
|
||||
const st = await fetchStatus();
|
||||
if (!st.ok) return;
|
||||
const rows = listServers();
|
||||
const byName = new Map(rows.map((r) => [r.name, r]));
|
||||
// For the dashboard: emit a lightweight snapshot. Actual per-server
|
||||
// resolution is handled by playersOn() during a sweep, which caches
|
||||
// iw4m_match/value into the servers table.
|
||||
emit({
|
||||
type: 'server-status',
|
||||
entries: st.entries.map((e) => ({
|
||||
server: e.name ?? String(e.port ?? ''),
|
||||
players: e.currentPlayers ?? null,
|
||||
name: e.name
|
||||
}))
|
||||
});
|
||||
// Also fold current auth ttl into the auth channel so subscribers get a
|
||||
// heartbeat and a countdown without waiting for a transition.
|
||||
const auth = getAuthState();
|
||||
emit({
|
||||
type: 'auth-state',
|
||||
state: auth,
|
||||
msRemaining: auth.kind === 'ready' ? Math.max(0, auth.expiresAt - Date.now()) : undefined
|
||||
});
|
||||
void byName; // reserved: future join between /api/status and server registry
|
||||
}
|
||||
|
||||
export function startWorker(): void {
|
||||
if (started) return;
|
||||
started = true;
|
||||
|
||||
// Seed the servers table from the fs so first-boot has rows to reference.
|
||||
seedServerRegistry().catch((err) => console.error('[worker] seed error:', err));
|
||||
|
||||
void tick();
|
||||
void pushServerStatus();
|
||||
tickTimer = setInterval(tick, TICK_MS);
|
||||
statusTimer = setInterval(pushServerStatus, STATUS_REFRESH_MS);
|
||||
|
||||
console.log('[worker] started — sweep every', TICK_MS, 'ms, status every', STATUS_REFRESH_MS, 'ms');
|
||||
}
|
||||
|
||||
export function stopWorker(): void {
|
||||
if (tickTimer) clearInterval(tickTimer);
|
||||
if (statusTimer) clearInterval(statusTimer);
|
||||
tickTimer = null;
|
||||
statusTimer = null;
|
||||
started = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* External trigger — called from `/api/push` and beta endpoints so a sweep
|
||||
* fires immediately after a state change instead of waiting for the next tick.
|
||||
*/
|
||||
export async function kick(): Promise<void> {
|
||||
await tick();
|
||||
}
|
||||
Reference in New Issue
Block a user