josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/beta-evaluation.ts
+81 -26
View File
@@ -19,14 +19,30 @@ export interface TestMetrics {
outages: number;
streaks: Record<string, number>;
}
export const emptyMetrics = (): TestMetrics => ({ values: {}, manual: {}, outages: 0, streaks: {} });
export const emptyMetrics = (): TestMetrics => ({
values: {},
manual: {},
outages: 0,
streaks: {}
});
/** Gameplay comes only from the script. No reporter IDs or beta handshake. */
export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: GameEvent, now: number): void {
export function applyGameEvent(
plan: BetaPlan,
metrics: TestMetrics,
event: GameEvent,
now: number
): void {
const players = event.players.length;
if (event.event === 'game_start' || !metrics.match || metrics.match.map !== event.map) {
metrics.match = { map: event.map, started: event.event === 'game_start' ? now : undefined,
lastEvent: now, players, ended: false, rounds: {} };
metrics.match = {
map: event.map,
started: event.event === 'game_start' ? now : undefined,
lastEvent: now,
players,
ended: false,
rounds: {}
};
metrics.streaks = {};
}
const match = metrics.match;
@@ -34,10 +50,12 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game
const minutes = Math.max(0, now - match.lastEvent) / 60_000;
for (const r of plan.requirements) {
const current = metrics.values[r.id] ?? 0;
const populated = players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers);
const populated =
players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers);
if (r.kind === 'population') metrics.values[r.id] = Math.max(current, players);
if (r.kind === 'populated-minutes' && populated) metrics.values[r.id] = current + minutes;
if (r.kind === 'player-hours' && populated) metrics.values[r.id] = current + minutes * Math.min(players, match.players) / 60;
if (r.kind === 'player-hours' && populated)
metrics.values[r.id] = current + (minutes * Math.min(players, match.players)) / 60;
if (r.kind === 'sustained-population') {
metrics.streaks[r.id] = populated ? (metrics.streaks[r.id] ?? 0) + minutes : 0;
metrics.values[r.id] = Math.max(current, metrics.streaks[r.id]);
@@ -46,10 +64,14 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game
match.rounds[r.id] = Math.max(match.rounds[r.id] ?? 0, event.round);
if (r.kind === 'round') metrics.values[r.id] = Math.max(current, event.round);
}
if (event.event === 'game_end' && r.kind === 'matches'
&& event.round >= MIN_RECORDED_ROUND
&& (match.rounds[r.id] ?? -1) >= r.minRound
&& (r.minMinutes === 0 || (match.started !== undefined && now - match.started >= r.minMinutes * 60_000))) {
if (
event.event === 'game_end' &&
r.kind === 'matches' &&
event.round >= MIN_RECORDED_ROUND &&
(match.rounds[r.id] ?? -1) >= r.minRound &&
(r.minMinutes === 0 ||
(match.started !== undefined && now - match.started >= r.minMinutes * 60_000))
) {
metrics.values[r.id] = current + 1;
}
}
@@ -66,30 +88,63 @@ export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy:
const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
if (!healthy && m.wasHealthy === true) m.outages++;
for (const r of plan.requirements) {
if (r.kind === 'healthy-minutes') m.values[r.id] = healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0;
if (r.kind === 'healthy-minutes')
m.values[r.id] =
healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0;
}
m.lastSample = at; m.wasHealthy = healthy;
m.lastSample = at;
m.wasHealthy = healthy;
}
export function evaluateRequirements(plan: BetaPlan, metrics: TestMetrics, phase: number): RequirementProgress[] {
return plan.requirements.map(r => {
export function evaluateRequirements(
plan: BetaPlan,
metrics: TestMetrics,
phase: number
): RequirementProgress[] {
return plan.requirements.map((r) => {
const budget = r.kind === 'failure-budget';
const current = budget ? metrics.outages : r.kind === 'manual' ? Number(!!metrics.manual[r.id])
: r.kind === 'recovery' ? phase : metrics.values[r.id] ?? 0;
return { id: r.id, label: r.label, current, target: r.target, passed: budget ? current <= r.target : current >= r.target,
const current = budget
? metrics.outages
: r.kind === 'manual'
? Number(!!metrics.manual[r.id])
: r.kind === 'recovery'
? phase
: (metrics.values[r.id] ?? 0);
return {
id: r.id,
label: r.label,
current,
target: r.target,
passed: budget ? current <= r.target : current >= r.target,
failed: budget && current > r.target,
detail: r.kind === 'manual' ? metrics.manual[r.id] ?? 'Awaiting tester sign-off'
: budget ? `${current} / ${r.target} allowed; monitored until promotion`
: `${Math.round(current * 100) / 100} / ${r.target}` };
detail:
r.kind === 'manual'
? (metrics.manual[r.id] ?? 'Awaiting tester sign-off')
: budget
? `${current} / ${r.target} allowed; monitored until promotion`
: `${Math.round(current * 100) / 100} / ${r.target}`
};
});
}
export function coveragePassed(plan: BetaPlan, servers: Array<{ passed: boolean; game: string | null; map: string | null }>): boolean {
export function coveragePassed(
plan: BetaPlan,
servers: Array<{ passed: boolean; game: string | null; map: string | null }>
): boolean {
if (!servers.length) return false;
if (plan.scope === 'all') return servers.every(s => s.passed);
if (plan.scope === 'quorum') return servers.filter(s => s.passed).length >= plan.quorum;
const key = (s: typeof servers[number]) => plan.scope === 'game' ? s.game : plan.scope === 'map' ? s.map
: s.game && s.map ? `${s.game}/${s.map}` : null;
if (plan.scope === 'all') return servers.every((s) => s.passed);
if (plan.scope === 'quorum') return servers.filter((s) => s.passed).length >= plan.quorum;
const key = (s: (typeof servers)[number]) =>
plan.scope === 'game'
? s.game
: plan.scope === 'map'
? s.map
: s.game && s.map
? `${s.game}/${s.map}`
: null;
const groups = new Set(servers.map(key));
return !groups.has(null) && [...groups].every(group => servers.some(s => s.passed && key(s) === group));
return (
!groups.has(null) &&
[...groups].every((group) => servers.some((s) => s.passed && key(s) === group))
);
}
src/lib/server/fleet/beta-pages.ts
+39 -12
View File
@@ -8,13 +8,21 @@ const positive = (value: string | null, fallback: number) => {
};
export function betaListPage(params: URLSearchParams) {
const state = params.get('state'), view = params.get('view'), search = (params.get('q') ?? '').slice(0, 200).trim();
const predicates: string[] = [], args: (string | number)[] = [];
if (state) { predicates.push('b.state=?'); args.push(state); }
else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')");
const state = params.get('state'),
view = params.get('view'),
search = (params.get('q') ?? '').slice(0, 200).trim();
const predicates: string[] = [],
args: (string | number)[] = [];
if (state) {
predicates.push('b.state=?');
args.push(state);
} else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')");
else if (view === 'history') predicates.push("b.state IN ('promoted','rolled-back')");
const status = params.get('status');
if (status && ['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status)) {
if (
status &&
['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status)
) {
// Match betaStatus: closed lifecycle wins over the final test outcome.
predicates.push(`(CASE WHEN b.state IN ('promoted','rolled-back') THEN b.state
WHEN EXISTS (SELECT 1 FROM beta_servers m WHERE m.beta_id=b.id AND (m.server_deleted_at IS NOT NULL OR m.stage='failed'))
@@ -24,8 +32,15 @@ export function betaListPage(params: URLSearchParams) {
args.push(status);
}
if (search) {
const names = (db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[])
.filter(row => `${row.server} ${serverTitle(row.server)}`.toLowerCase().includes(search.toLowerCase())).map(row => row.server);
const names = (
db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[]
)
.filter((row) =>
`${row.server} ${serverTitle(row.server)}`
.toLowerCase()
.includes(search.toLowerCase())
)
.map((row) => row.server);
predicates.push(`(instr(lower('#'||b.id||' '||COALESCE(b.notes,'')),lower(?))>0 OR EXISTS (
SELECT 1 FROM beta_scripts bs JOIN scripts s ON s.id=bs.script_id WHERE bs.beta_id=b.id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0))
OR EXISTS (SELECT 1 FROM scripts s WHERE s.id=b.script_id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0))
@@ -33,21 +48,33 @@ export function betaListPage(params: URLSearchParams) {
args.push(search, search, search, search, search, JSON.stringify(names));
}
const where = predicates.length ? ' WHERE ' + predicates.join(' AND ') : '';
const total = (db().prepare('SELECT COUNT(*) n FROM betas b' + where).get(...args) as { n: number }).n;
const total = (
db()
.prepare('SELECT COUNT(*) n FROM betas b' + where)
.get(...args) as { n: number }
).n;
const pageSize = Math.min(100, positive(params.get('limit'), 25));
const totalPages = Math.max(1, Math.ceil(total / pageSize));
const page = Math.min(totalPages, positive(params.get('page'), 1));
const betas = db().prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?')
const betas = db()
.prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?')
.all(...args, pageSize, (page - 1) * pageSize) as BetaRow[];
const counts = db().prepare(`SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active,
COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas`).get() as { active: number; history: number };
const counts = db()
.prepare(
`SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active,
COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas`
)
.get() as { active: number; history: number };
return { betas, page, pageSize, total, totalPages, counts };
}
export function betaActivityPage(id: number, params = new URLSearchParams()) {
const before = positive(params.get('before'), Number.MAX_SAFE_INTEGER);
const limit = Math.min(100, positive(params.get('limit'), 50));
const rows = db().prepare('SELECT id,created_at,message FROM beta_events WHERE beta_id=? AND id<? ORDER BY id DESC LIMIT ?')
const rows = db()
.prepare(
'SELECT id,created_at,message FROM beta_events WHERE beta_id=? AND id<? ORDER BY id DESC LIMIT ?'
)
.all(id, before, limit + 1) as { id: number; created_at: string; message: string }[];
const more = rows.length > limit;
const events = rows.slice(0, limit).reverse();
src/lib/server/fleet/beta-revisions.ts
+51 -13
View File
@@ -2,29 +2,67 @@ import { db } from '../db/index.js';
import type { BetaRevision } from '../../beta-revision.js';
export function currentBetaRevision(id: number): { revision: number; started_at: string } {
return db().prepare(`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`).get(id) as
{ revision: number; started_at: string } | undefined
?? { revision: 1, started_at: (db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as { opened_at: string }).opened_at };
return (
(db()
.prepare(
`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`
)
.get(id) as { revision: number; started_at: string } | undefined) ?? {
revision: 1,
started_at: (
db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as {
opened_at: string;
}
).opened_at
}
);
}
/** Called in the revision transaction, before replacing the pinned bundle. */
export function archiveBetaRevision(id: number, revision: number, startedAt: string, notes: string | null): void {
db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)')
export function archiveBetaRevision(
id: number,
revision: number,
startedAt: string,
notes: string | null
): void {
db()
.prepare(
'INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)'
)
.run(id, revision, startedAt, notes);
db().prepare(`INSERT OR IGNORE INTO beta_revision_files
db()
.prepare(
`INSERT OR IGNORE INTO beta_revision_files
(beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content_hash)
SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content_hash
FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id
WHERE bs.beta_id = ?`).run(revision, id);
WHERE bs.beta_id = ?`
)
.run(revision, id);
}
export function betaRevisionHistory(id: number): BetaRevision[] {
const rows = db().prepare('SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC')
.all(id) as Array<{ revision: number; started_at: string; notes: string | null; results_json: string | null }>;
const files = db().prepare(`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`).all(id) as Array<BetaRevision['files'][number] & { revision: number }>;
return rows.map(({ results_json, ...row }) => ({ ...row,
const rows = db()
.prepare(
'SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC'
)
.all(id) as Array<{
revision: number;
started_at: string;
notes: string | null;
results_json: string | null;
}>;
const files = db()
.prepare(
`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`
)
.all(id) as Array<BetaRevision['files'][number] & { revision: number }>;
return rows.map(({ results_json, ...row }) => ({
...row,
results: results_json ? JSON.parse(results_json) : null,
files: files.filter(file => file.revision === row.revision).map(({ revision, ...file }) => file)
files: files
.filter((file) => file.revision === row.revision)
.map(({ revision, ...file }) => file)
}));
}
src/lib/server/fleet/beta-selection.ts
+103 -40
View File
@@ -7,7 +7,7 @@ import { observationReader } from './status.js';
import type { ScriptRow } from './scripts.js';
const games = ['t5', 't6'] as const;
type Game = typeof games[number];
type Game = (typeof games)[number];
type Counts = Partial<Record<Game, number>>;
export function parseBetaInstanceCounts(value: unknown): Counts {
@@ -16,7 +16,12 @@ export function parseBetaInstanceCounts(value: unknown): Counts {
}
const counts: Counts = {};
for (const [game, count] of Object.entries(value)) {
if (!games.includes(game as Game) || !Number.isSafeInteger(count) || count < 1 || count > 1000) {
if (
!games.includes(game as Game) ||
!Number.isSafeInteger(count) ||
count < 1 ||
count > 1000
) {
throw new Error('Instance counts must be integers from 1 to 1000 for t5 or t6');
}
counts[game as Game] = count;
@@ -25,79 +30,137 @@ export function parseBetaInstanceCounts(value: unknown): Counts {
}
function matches(script: ScriptRow, server: ServerRow): boolean {
return !script.scope_type || !script.scope_value
|| (script.scope_type === 'game' ? server.game : server.map) === script.scope_value;
return (
!script.scope_type ||
!script.scope_value ||
(script.scope_type === 'game' ? server.game : server.map) === script.scope_value
);
}
/** Choose at submission time using the same fresh observations shown in Fleet. */
export function selectBetaServers(scripts: ScriptRow[], instancesPerGame: unknown): string[] {
const counts = parseBetaInstanceCounts(instancesPerGame), observe = observationReader(), now = Date.now();
const hosts = new Map(listHosts().map(host => [host.id, host]));
const busy = new Set((db().prepare(`SELECT server FROM server_deployments WHERE server_deleted_at IS NULL
const counts = parseBetaInstanceCounts(instancesPerGame),
observe = observationReader(),
now = Date.now();
const hosts = new Map(listHosts().map((host) => [host.id, host]));
const busy = new Set(
(
db()
.prepare(
`SELECT server FROM server_deployments WHERE server_deleted_at IS NULL
AND (state IN ('queued','waiting','running','awaiting-health','interrupted')
OR (action IN ('destroy','rebuild') AND state = 'failed'
AND stage IN ('Stopping and removing container','Removing server files and registry')))`)
.all() as { server: string }[]).map(row => row.server));
const candidates = listServers().flatMap(server => {
if (!counts[server.game as Game] || server.desired_state === 'stopped' || busy.has(server.name)
|| !scripts.some(script => matches(script, server)) || protectedWork(server.name)) return [];
const host = server.host_id ? hosts.get(server.host_id) : null;
if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission)) return [];
const observation = observe(server);
if (observation.health !== 'online' || observation.last_players === null
|| !freshStatusTimestamp(observation.last_seen_at, now)) return [];
// An occupied RCON observation must not rank as empty just because a
// competing telemetry report claims zero players.
const rconPlayers = freshStatusTimestamp(server.last_seen_at, now)
&& (!observation.telemetry_after || Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after))
? server.last_players ?? 0 : 0;
return [{ server, players: Math.max(observation.last_players, rconPlayers) }];
}).sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name));
AND stage IN ('Stopping and removing container','Removing server files and registry')))`
)
.all() as { server: string }[]
).map((row) => row.server)
);
const candidates = listServers()
.flatMap((server) => {
if (
!counts[server.game as Game] ||
server.desired_state === 'stopped' ||
busy.has(server.name) ||
!scripts.some((script) => matches(script, server)) ||
protectedWork(server.name)
)
return [];
const host = server.host_id ? hosts.get(server.host_id) : null;
if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission))
return [];
const observation = observe(server);
if (
observation.health !== 'online' ||
observation.last_players === null ||
!freshStatusTimestamp(observation.last_seen_at, now)
)
return [];
// An occupied RCON observation must not rank as empty just because a
// competing telemetry report claims zero players.
const rconPlayers =
freshStatusTimestamp(server.last_seen_at, now) &&
(!observation.telemetry_after ||
Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after))
? (server.last_players ?? 0)
: 0;
return [{ server, players: Math.max(observation.last_players, rconPlayers) }];
})
.sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name));
for (const game of games) {
const available = candidates.filter(candidate => candidate.server.game === game).length;
const available = candidates.filter((candidate) => candidate.server.game === game).length;
if ((counts[game] ?? 0) > available) {
throw new Error(`${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.`);
throw new Error(
`${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.`
);
}
}
for (const script of scripts) {
if (!candidates.some(candidate => matches(script, candidate.server))) {
throw new Error(`${script.name}: no available server in the requested games matches this file's scope.`);
if (!candidates.some((candidate) => matches(script, candidate.server))) {
throw new Error(
`${script.name}: no available server in the requested games matches this file's scope.`
);
}
}
// Map scopes need a representative even when the least populated servers
// all run the same map. Reserve enough remaining slots for every map.
const requiredMaps = new Set(scripts.filter(script => script.scope_type === 'map').map(script => script.scope_value!));
const requiredMaps = new Set(
scripts.filter((script) => script.scope_type === 'map').map((script) => script.scope_value!)
);
function canComplete(remaining: typeof candidates, slots: Counts, maps: Set<string>): boolean {
const options = new Map<string, Set<Game>>();
for (const game of games) {
if (remaining.filter(candidate => candidate.server.game === game).length < (slots[game] ?? 0)) return false;
if (
remaining.filter((candidate) => candidate.server.game === game).length <
(slots[game] ?? 0)
)
return false;
}
for (const map of maps) {
const choices = new Set(remaining.filter(candidate => candidate.server.map === map && (slots[candidate.server.game as Game] ?? 0) > 0)
.map(candidate => candidate.server.game as Game));
const choices = new Set(
remaining
.filter(
(candidate) =>
candidate.server.map === map &&
(slots[candidate.server.game as Game] ?? 0) > 0
)
.map((candidate) => candidate.server.game as Game)
);
if (!choices.size) return false;
options.set(map, choices);
}
// There are two supported games: single-game maps reserve that game's
// slots, and maps available in either game use any remaining slot.
return games.every(game => [...options.values()].filter(choices => choices.size === 1 && choices.has(game)).length <= (slots[game] ?? 0))
&& maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0);
return (
games.every(
(game) =>
[...options.values()].filter(
(choices) => choices.size === 1 && choices.has(game)
).length <= (slots[game] ?? 0)
) && maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0)
);
}
if (!canComplete(candidates, counts, requiredMaps)) {
throw new Error('Increase the instance counts to cover every selected file\'s map scope, or choose servers in advanced options.');
throw new Error(
"Increase the instance counts to cover every selected file's map scope, or choose servers in advanced options."
);
}
const selected: string[] = [], slots = { ...counts };
const selected: string[] = [],
slots = { ...counts };
let maps = requiredMaps;
for (let index = 0; index < candidates.length; index++) {
const { server } = candidates[index], game = server.game as Game;
const { server } = candidates[index],
game = server.game as Game;
if (!slots[game]) continue;
const nextSlots = { ...slots, [game]: slots[game]! - 1 }, nextMaps = new Set(maps);
const nextSlots = { ...slots, [game]: slots[game]! - 1 },
nextMaps = new Set(maps);
if (server.map) nextMaps.delete(server.map);
if (!canComplete(candidates.slice(index + 1), nextSlots, nextMaps)) continue;
selected.push(server.name); slots[game] = nextSlots[game]; maps = nextMaps;
if (games.every(game => !slots[game])) break;
selected.push(server.name);
slots[game] = nextSlots[game];
maps = nextMaps;
if (games.every((game) => !slots[game])) break;
}
return selected;
}
src/lib/server/fleet/beta-tests.ts
+145 -36
View File
@@ -4,52 +4,120 @@ import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
import type { GameTelemetry } from '../../telemetry.js';
import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
import {
applySample,
applyGameEvent,
emptyMetrics,
evaluateRequirements,
type TestMetrics
} from './beta-evaluation.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { currentBetaRevision } from './beta-revisions.js';
import { invalidateBeta } from '../pipeline/bus.js';
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
export interface TestRun {
id: string;
beta_id: number;
server: string;
phase: number;
created_at: string;
observing_at: string | null;
metrics: string;
}
export interface StoredPlan {
beta_id: number;
plan: string;
outcome: string;
reason: string | null;
}
const nowIso = () => new Date().toISOString();
const activeParticipant = (id: number, server: string) => !!db().prepare(`SELECT 1 FROM beta_servers
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`).get(id, server);
export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined;
export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined;
export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); invalidateBeta(id); }
const activeParticipant = (id: number, server: string) =>
!!db()
.prepare(
`SELECT 1 FROM beta_servers
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
)
.get(id, server);
export const storedPlan = (id: number) =>
db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as
StoredPlan | undefined;
export const currentRun = (id: number, server: string) =>
db()
.prepare(
'SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL'
)
.get(id, server) as TestRun | undefined;
export function activity(id: number, message: string) {
db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message);
invalidateBeta(id);
}
export function setOutcome(id: number, outcome: string, reason: string | null): void {
const previous = storedPlan(id);
if (!previous || (previous.outcome === outcome && previous.reason === reason)) return;
db().prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?').run(outcome, reason, id);
db()
.prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?')
.run(outcome, reason, id);
activity(id, `Test plan ${outcome}${reason ? `: ${reason}` : ''}`);
}
export function readMetrics(run: TestRun): TestMetrics { return { ...emptyMetrics(), ...JSON.parse(run.metrics) }; }
export function saveMetrics(run: TestRun, metrics: TestMetrics): void { db().prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?').run(JSON.stringify(metrics), run.id); invalidateBeta(run.beta_id); }
export function readMetrics(run: TestRun): TestMetrics {
return { ...emptyMetrics(), ...JSON.parse(run.metrics) };
}
export function saveMetrics(run: TestRun, metrics: TestMetrics): void {
db()
.prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?')
.run(JSON.stringify(metrics), run.id);
invalidateBeta(run.beta_id);
}
export function evidence(run: TestRun, eventId: string, kind: string, payload: unknown): boolean {
return db().prepare('INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)')
.run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0;
return (
db()
.prepare(
'INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)'
)
.run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0
);
}
/** Internal progress record; never sent to or required from the game script. */
export async function prepareRun(betaId: number, server: string, phase = 0): Promise<TestRun> {
if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta');
if (!activeParticipant(betaId, server))
throw new Error('Original test server was deleted; start a new beta');
db().transaction(() => {
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server);
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), currentBetaRevision(betaId).revision);
db()
.prepare(
'UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL'
)
.run(nowIso(), betaId, server);
db()
.prepare(
'INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)'
)
.run(
randomUUID(),
betaId,
server,
phase,
nowIso(),
JSON.stringify(emptyMetrics()),
currentBetaRevision(betaId).revision
);
})();
return currentRun(betaId, server)!;
}
/** Route ordinary server events to every deployed, active beta automatically. */
export function recordGameEvent(event: GameTelemetry): void {
const runs = db().prepare(`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
const runs = db()
.prepare(
`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
JOIN beta_test_plans p ON p.beta_id = b.id
JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server
WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote')
AND s.server_deleted_at IS NULL
AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready')
AND s.restarted_at IS NOT NULL`).all(event.server) as TestRun[];
AND s.restarted_at IS NOT NULL`
)
.all(event.server) as TestRun[];
for (const run of runs) {
const plan = parseBetaPlan(JSON.parse(storedPlan(run.beta_id)!.plan));
const metrics = readMetrics(run);
@@ -64,17 +132,27 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
const m = readMetrics(run);
const row = getServer(run.server);
const server = row ? serverObservation(row) : null;
const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
const timestamp =
server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
const sample = Date.parse(timestamp ?? '');
const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
&& Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS;
const freshSample =
Number.isFinite(sample) &&
sample >= Date.parse(run.observing_at ?? run.created_at) &&
Date.now() - sample >= 0 &&
Date.now() - sample <= ONLINE_MAX_MS;
const healthy = freshSample && server?.health === 'online';
const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
const before = m.wasHealthy;
const lastSample = m.lastSample;
applySample(plan, m, at, healthy);
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, source: server?.status_source });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
if (freshSample && sample !== lastSample)
evidence(run, `sample:${sample}`, 'status-sample', {
healthy,
checkedAt: timestamp,
source: server?.status_source
});
if (before !== m.wasHealthy)
evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
saveMetrics(run, m);
}
@@ -82,22 +160,53 @@ export function testView(betaId: number): BetaTestView | null {
const stored = storedPlan(betaId);
if (!stored) return null;
const plan = parseBetaPlan(JSON.parse(stored.plan));
const runs = db().prepare(`SELECT r.*, s.server_deleted_at FROM beta_test_runs r
const runs = db()
.prepare(
`SELECT r.*, s.server_deleted_at FROM beta_test_runs r
JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`).all(betaId) as Array<TestRun & { server_deleted_at: string | null }>;
return { plan, outcome: stored.outcome, reason: stored.reason,
servers: runs.map(run => {
WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`
)
.all(betaId) as Array<TestRun & { server_deleted_at: string | null }>;
return {
plan,
outcome: stored.outcome,
reason: stored.reason,
servers: runs.map((run) => {
const metrics = readMetrics(run);
const progress = evaluateRequirements(plan, metrics, run.phase);
const latest = db().prepare("SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1").get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
const latest = db()
.prepare(
"SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1"
)
.get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
const event = latest ? JSON.parse(latest.payload) : null;
return { server: run.server, server_deleted_at: run.server_deleted_at, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at,
observation: { lastGameplayAt: latest?.received_at ?? null, lastEvent: latest?.kind ?? null,
players: metrics.match?.players ?? null, round: event?.round ?? null,
matchStarted: metrics.match?.started !== undefined, gameEnded: metrics.match?.ended ?? false,
healthy: metrics.wasHealthy ?? null, healthCheckedAt: metrics.lastSample ? new Date(metrics.lastSample).toISOString() : null } };
return {
server: run.server,
server_deleted_at: run.server_deleted_at,
phase: run.phase,
passed: progress.every((r) => r.passed),
progress,
observingAt: run.observing_at,
observation: {
lastGameplayAt: latest?.received_at ?? null,
lastEvent: latest?.kind ?? null,
players: metrics.match?.players ?? null,
round: event?.round ?? null,
matchStarted: metrics.match?.started !== undefined,
gameEnded: metrics.match?.ended ?? false,
healthy: metrics.wasHealthy ?? null,
healthCheckedAt: metrics.lastSample
? new Date(metrics.lastSample).toISOString()
: null
}
};
}),
evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
evidence: db()
.prepare(
`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`).all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence'] };
WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`
)
.all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence']
};
}
src/lib/server/fleet/beta.ts
+783 -251
View File
File diff suppressed because it is too large Load Diff
src/lib/server/fleet/chat.ts
+184 -49
View File
@@ -1,32 +1,61 @@
import { createHash } from 'node:crypto';
import { db } from '../db/index.js';
import { enrichGameChatTimeline, type ChatMatch, type ChatRoundReport, type ContextualChatEntry } from '../../game-chat.js';
import {
enrichGameChatTimeline,
type ChatMatch,
type ChatRoundReport,
type ContextualChatEntry
} from '../../game-chat.js';
import { parseGameEvent } from '../../telemetry.js';
import { telemetryRetention, telemetryCutoff } from './telemetry-requests.js';
import { getServer, listServers, type ServerRow } from './servers.js';
import { gameLogKey, resolveLogPath, readLogChunk, type LogCursor, type TailResult } from './logs.js';
import {
gameLogKey,
resolveLogPath,
readLogChunk,
type LogCursor,
type TailResult
} from './logs.js';
import { listHosts } from '../hosts/index.js';
import { observationReader } from './status.js';
/** Historical context is scoped to this registration, including round-one matches. */
export function serverChatTimeline(server: ServerRow, tail: TailResult): ContextualChatEntry[] {
const unscoped = enrichGameChatTimeline(tail.lines, { mtime: tail.mtime, matches: [], reports: [] });
const times = unscoped.flatMap(entry => entry.context.realTime ? [entry.context.realTime] : []);
const unscoped = enrichGameChatTimeline(tail.lines, {
mtime: tail.mtime,
matches: [],
reports: []
});
const times = unscoped.flatMap((entry) =>
entry.context.realTime ? [entry.context.realTime] : []
);
if (!times.length) return unscoped;
const since = times.reduce((a, b) => a < b ? a : b), until = times.reduce((a, b) => a > b ? a : b);
const since = times.reduce((a, b) => (a < b ? a : b)),
until = times.reduce((a, b) => (a > b ? a : b));
const handle = db();
const matches = handle.prepare(`SELECT id, started_at AS startedAt, ended_at AS endedAt,
const matches = handle
.prepare(
`SELECT id, started_at AS startedAt, ended_at AS endedAt,
last_received_at AS lastReceivedAt, status, start_observed AS startObserved
FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL
AND started_at <= ? AND (COALESCE(ended_at,last_received_at) >= ? OR status='active')
ORDER BY id DESC LIMIT 2000`).all(server.name, until, since) as Array<Omit<ChatMatch, 'startObserved'> & { startObserved: number }>;
ORDER BY id DESC LIMIT 2000`
)
.all(server.name, until, since) as Array<
Omit<ChatMatch, 'startObserved'> & { startObserved: number }
>;
// Diagnostic history survives deletion. Its receipt time must belong to the
// current registration so a reused server name cannot inherit old rounds.
const cutoff = telemetryCutoff();
const roundSince = new Date(Date.parse(since) - 120_000).toISOString();
const matchSince = matches.reduce((earliest, match) => match.startedAt < earliest ? match.startedAt : earliest, since);
const reports = handle.prepare(`SELECT received_at AS receivedAt, event, round,
const matchSince = matches.reduce(
(earliest, match) => (match.startedAt < earliest ? match.startedAt : earliest),
since
);
const reports = handle
.prepare(
`SELECT received_at AS receivedAt, event, round,
player_count AS playerCount FROM telemetry_requests
WHERE server = ? AND status = 200 AND received_at >= ?
AND julianday(received_at) >= julianday(?)
@@ -34,36 +63,69 @@ export function serverChatTimeline(server: ServerRow, tail: TailResult): Context
AND received_at <= ?
AND event IN ('game_start', 'round_change', 'heartbeat', 'game_end')
AND player_count >= 0 AND (event = 'game_start' OR round >= 0)
ORDER BY id DESC LIMIT ?`).all(server.name, cutoff, server.created_at, roundSince, matchSince,
ORDER BY id DESC LIMIT ?`
)
.all(
server.name,
cutoff,
server.created_at,
roundSince,
matchSince,
new Date(Date.parse(until) + 120_000).toISOString(),
(telemetryRetention().maxPerServer || 10000)) as ChatRoundReport[];
telemetryRetention().maxPerServer || 10000
) as ChatRoundReport[];
// Saving gameplay does not depend on diagnostic logging succeeding. Keep the
// surviving accepted snapshot useful when its request log was not recorded.
const latest = handle.prepare(`SELECT received_at AS receivedAt, payload FROM server_telemetry
WHERE server = ? AND received_at >= ? AND julianday(received_at) >= julianday(?)`)
.get(server.name, cutoff, server.created_at) as { receivedAt: string; payload: string | null } | undefined;
const latest = handle
.prepare(
`SELECT received_at AS receivedAt, payload FROM server_telemetry
WHERE server = ? AND received_at >= ? AND julianday(received_at) >= julianday(?)`
)
.get(server.name, cutoff, server.created_at) as
{ receivedAt: string; payload: string | null } | undefined;
if (latest?.payload) {
try {
const event = parseGameEvent(JSON.parse(latest.payload));
const round = event.event === 'game_start' ? null : event.round;
if (event.server === server.name && !reports.some(report => report.receivedAt === latest.receivedAt
&& report.event === event.event && report.round === round && report.playerCount === event.players.length)) {
reports.push({ receivedAt: latest.receivedAt, event: event.event, round, playerCount: event.players.length });
if (
event.server === server.name &&
!reports.some(
(report) =>
report.receivedAt === latest.receivedAt &&
report.event === event.event &&
report.round === round &&
report.playerCount === event.players.length
)
) {
reports.push({
receivedAt: latest.receivedAt,
event: event.event,
round,
playerCount: event.players.length
});
}
} catch { /* Legacy malformed snapshots cannot establish chat context. */ }
} catch {
/* Legacy malformed snapshots cannot establish chat context. */
}
}
return enrichGameChatTimeline(tail.lines, {
mtime: tail.mtime,
matches: matches.map(match => ({ ...match, startObserved: Boolean(match.startObserved) })),
matches: matches.map((match) => ({
...match,
startObserved: Boolean(match.startObserved)
})),
reports
});
}
export function matchChatTimeline(matchId: number): ContextualChatEntry[] {
return (db().prepare('SELECT entry FROM match_chat WHERE match_id = ? ORDER BY rowid')
.all(matchId) as { entry: string }[]).map(row => JSON.parse(row.entry));
return (
db()
.prepare('SELECT entry FROM match_chat WHERE match_id = ? ORDER BY rowid')
.all(matchId) as { entry: string }[]
).map((row) => JSON.parse(row.entry));
}
/** Save before filtering the live view so completed matches retain their messages. */
@@ -86,15 +148,24 @@ function archiveChat(entries: ContextualChatEntry[]): void {
})();
}
export function currentMatchChat(server: ServerRow, tail: TailResult | null): ContextualChatEntry[] {
export function currentMatchChat(
server: ServerRow,
tail: TailResult | null
): ContextualChatEntry[] {
const entries = tail ? serverChatTimeline(server, tail) : [];
archiveChat(entries);
return currentChatEntries(server, entries);
}
function currentChatEntries(server: ServerRow, entries: ContextualChatEntry[]): ContextualChatEntry[] {
const match = db().prepare(`SELECT id, started_at AS startedAt FROM leaderboard_matches
WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1`)
function currentChatEntries(
server: ServerRow,
entries: ContextualChatEntry[]
): ContextualChatEntry[] {
const match = db()
.prepare(
`SELECT id, started_at AS startedAt FROM leaderboard_matches
WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1`
)
.get(server.name) as { id: number; startedAt: string } | undefined;
let startIndex = entries.length - 1;
while (startIndex >= 0) {
@@ -105,27 +176,44 @@ function currentChatEntries(server: ServerRow, entries: ContextualChatEntry[]):
const current = entries.slice(Math.max(0, startIndex));
if (!match) {
// Servers without gameplay telemetry still have log-defined sessions.
return current.filter(entry => entry.context.realTime !== null
&& Date.parse(entry.context.realTime) >= Date.parse(server.created_at));
return current.filter(
(entry) =>
entry.context.realTime !== null &&
Date.parse(entry.context.realTime) >= Date.parse(server.created_at)
);
}
const start = entries[startIndex];
// A log restart can arrive before game_start telemetry. Clear immediately
// instead of falling back to messages from the previous recorded match.
if (start && !current.some(entry => entry.matchId === match.id)
&& (!start.context.realTime || Date.parse(start.context.realTime) > Date.parse(match.startedAt) + 1000)) return [];
if (
start &&
!current.some((entry) => entry.matchId === match.id) &&
(!start.context.realTime ||
Date.parse(start.context.realTime) > Date.parse(match.startedAt) + 1000)
)
return [];
return matchChatTimeline(match.id);
}
const pending = new Map<string, Promise<{ entries: ContextualChatEntry[]; missing: boolean }>>();
interface ChatReadState {
key: string; cursor?: LogCursor; lines: string[]; partial: Buffer; skipLine: boolean;
signature?: string; entries: ContextualChatEntry[]; contextVersion?: string; wasOccupied: boolean;
key: string;
cursor?: LogCursor;
lines: string[];
partial: Buffer;
skipLine: boolean;
signature?: string;
entries: ContextualChatEntry[];
contextVersion?: string;
wasOccupied: boolean;
}
const readers = new Map<string, ChatReadState>();
const failures = new Map<string, number>();
/** Shared by the live API and the worker; collection does not require a viewer. */
export function collectServerChat(server: ServerRow): Promise<{ entries: ContextualChatEntry[]; missing: boolean }> {
export function collectServerChat(
server: ServerRow
): Promise<{ entries: ContextualChatEntry[]; missing: boolean }> {
const key = `${server.name}:${server.created_at}:${server.native_instance_id}:${server.instance_generation}`;
const existing = pending.get(key);
if (existing) return existing;
@@ -135,12 +223,26 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
let saved = readers.get(server.name);
if (saved?.key !== key) saved = undefined;
// Publish the new cursor only after its messages have been processed.
const state: ChatReadState = saved ? { ...saved } : { key, lines: [], partial: Buffer.alloc(0), skipLine: false, entries: [], wasOccupied: false };
const state: ChatReadState = saved
? { ...saved }
: {
key,
lines: [],
partial: Buffer.alloc(0),
skipLine: false,
entries: [],
wasOccupied: false
};
const tail = await readLogChunk(path, state.cursor);
const current = getServer(server.name);
// A remote read may finish after destruction or replacement.
if (!current || current.destroyed_at || current.created_at !== server.created_at
|| current.native_instance_id !== server.native_instance_id || current.instance_generation !== server.instance_generation) {
if (
!current ||
current.destroyed_at ||
current.created_at !== server.created_at ||
current.native_instance_id !== server.native_instance_id ||
current.instance_generation !== server.instance_generation
) {
return { entries: [], missing: true };
}
if (!tail) {
@@ -149,22 +251,40 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
}
let changed = false;
if ('cursor' in tail) {
if (tail.reset) { state.lines = []; state.partial = Buffer.alloc(0); state.skipLine = tail.truncated; state.entries = []; }
if (tail.reset) {
state.lines = [];
state.partial = Buffer.alloc(0);
state.skipLine = tail.truncated;
state.entries = [];
}
state.cursor = tail.cursor;
const bytes = Buffer.concat([state.partial, Buffer.from(tail.contentBase64, 'base64')]);
const end = bytes.lastIndexOf(10) + 1;
if (end) {
const lines = bytes.subarray(0, end).toString('utf8').split('\n').slice(0, -1).map(line => line.replace(/\r$/, ''));
const lines = bytes
.subarray(0, end)
.toString('utf8')
.split('\n')
.slice(0, -1)
.map((line) => line.replace(/\r$/, ''));
if (state.skipLine) lines.shift();
state.skipLine = false;
state.lines = [...state.lines, ...lines].slice(-2000);
let length = 0, start = state.lines.length;
while (start > 0 && (length += Buffer.byteLength(state.lines[start - 1]) + 1) <= 256 * 1024) start--;
let length = 0,
start = state.lines.length;
while (
start > 0 &&
(length += Buffer.byteLength(state.lines[start - 1]) + 1) <= 256 * 1024
)
start--;
state.lines = state.lines.slice(start);
changed = true;
}
state.partial = Buffer.from(bytes.subarray(end));
if (state.partial.length > 256 * 1024) { state.partial = Buffer.alloc(0); state.skipLine = true; }
if (state.partial.length > 256 * 1024) {
state.partial = Buffer.alloc(0);
state.skipLine = true;
}
// File mtime anchors the final line only after the backlog has been read.
if (tail.cursor.offset < tail.size || state.partial.length) {
state.signature = undefined;
@@ -177,9 +297,21 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
if (changed) state.lines = tail.lines;
}
const signature = `${tail.size}:${tail.mtime}`;
const needsContext = state.entries.some(entry => entry.type === 'message' && entry.matchId === undefined);
const contextVersion = needsContext ? (db().prepare('SELECT received_at FROM server_telemetry WHERE server=?').get(server.name) as { received_at: string } | undefined)?.received_at : undefined;
if (changed || state.signature === undefined || (needsContext && contextVersion !== state.contextVersion)) {
const needsContext = state.entries.some(
(entry) => entry.type === 'message' && entry.matchId === undefined
);
const contextVersion = needsContext
? (
db()
.prepare('SELECT received_at FROM server_telemetry WHERE server=?')
.get(server.name) as { received_at: string } | undefined
)?.received_at
: undefined;
if (
changed ||
state.signature === undefined ||
(needsContext && contextVersion !== state.contextVersion)
) {
state.entries = serverChatTimeline(current, { ...tail, lines: state.lines });
archiveChat(state.entries);
state.contextVersion = contextVersion;
@@ -193,15 +325,16 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
}
export async function collectFleetChat(): Promise<void> {
const hosts = new Map(listHosts().map(host => [host.id, host]));
const hosts = new Map(listHosts().map((host) => [host.id, host]));
const observe = observationReader();
const live = listServers().filter(server => !server.destroyed_at);
const names = new Set(live.map(server => server.name));
const live = listServers().filter((server) => !server.destroyed_at);
const names = new Set(live.map((server) => server.name));
for (const name of readers.keys()) if (!names.has(name)) readers.delete(name);
for (const name of failures.keys()) if (!names.has(name)) failures.delete(name);
const servers = live.filter(server => {
const servers = live.filter((server) => {
if (server.host_id && !hosts.get(server.host_id)?.connected) return false;
if (server.container_state && !['running', 'unknown'].includes(server.container_state)) return false;
if (server.container_state && !['running', 'unknown'].includes(server.container_state))
return false;
const players = observe(server).last_players ?? server.last_players;
// One final read after players leave retains messages written at match end.
return players !== 0 || readers.get(server.name)?.wasOccupied;
@@ -212,7 +345,9 @@ export async function collectFleetChat(): Promise<void> {
try {
await collectServerChat(server);
const state = readers.get(server.name);
if (state) state.wasOccupied = (observe(server).last_players ?? server.last_players ?? 0) > 0;
if (state)
state.wasOccupied =
(observe(server).last_players ?? server.last_players ?? 0) > 0;
failures.delete(server.name);
} catch (error) {
if (Date.now() - (failures.get(server.name) ?? -Infinity) >= 5 * 60_000) {
src/lib/server/fleet/collections.ts
+420 -129
View File
@@ -1,7 +1,12 @@
import { scalingOwnershipBlocker } from '../autoscaling/ownership.js';
import { fleetSettings } from '../settings.js';
import { randomUUID } from 'node:crypto';
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
import type {
CollectionDetail,
CollectionMode,
CollectionPreview,
CollectionSummary
} from '$lib/deployment-plan.js';
import { db } from '../db/index.js';
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
import { creationQueue, destroyServer } from '../provisioning/index.js';
@@ -18,97 +23,203 @@ import { canCancelCreation, cancelCreation } from '../provisioning/cancel-creati
import { canCancelRemoval, cancelRemoval } from '../provisioning/history.js';
function coverage(entries: BlueprintEntry[], includeHealth = true) {
type Server = { name: string; game: string | null; map_slug: string | null; map: string | null };
const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[];
const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
type Server = {
name: string;
game: string | null;
map_slug: string | null;
map: string | null;
};
const registered = db()
.prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL')
.all() as Server[];
const queued = db()
.prepare(
`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[];
const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server));
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`
)
.all() as Server[];
const removing = new Set(
(
db()
.prepare(
"SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')"
)
.all() as { server: string }[]
).map((row) => row.server)
);
// A provisioning server can appear in both tables; count its identity once.
const servers = [...new Map([...queued, ...registered].map(server => [server.name, server])).values()];
return entries.map(entry => {
const existingServers = servers.filter(server => {
if (removing.has(server.name)) return false;
const parsed = parseServerName(server.name);
const game = server.game ?? parsed?.game;
if (game !== entry.recipe.game) return false;
if (server.map_slug) return server.map_slug === entry.recipe.mapSlug;
const namedMap = parsed?.mapCanonical ? zombieMapBySlug(game, parsed.mapSlug) : undefined;
if (namedMap && (!server.map || server.map === namedMap.canonical)) return namedMap.slug === entry.recipe.mapSlug;
// Legacy canonical maps are usable only when they identify one variant.
const matches = zombieMaps(game).filter(map => map.canonical === server.map);
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
}).map(server => server.name).sort();
const coverage = includeHealth ? existingServers.map(name => {
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
: server ? serverObservation(server).health : 'offline',
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
}) : [];
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
const servers = [
...new Map([...queued, ...registered].map((server) => [server.name, server])).values()
];
return entries.map((entry) => {
const existingServers = servers
.filter((server) => {
if (removing.has(server.name)) return false;
const parsed = parseServerName(server.name);
const game = server.game ?? parsed?.game;
if (game !== entry.recipe.game) return false;
if (server.map_slug) return server.map_slug === entry.recipe.mapSlug;
const namedMap = parsed?.mapCanonical
? zombieMapBySlug(game, parsed.mapSlug)
: undefined;
if (namedMap && (!server.map || server.map === namedMap.canonical))
return namedMap.slug === entry.recipe.mapSlug;
// Legacy canonical maps are usable only when they identify one variant.
const matches = zombieMaps(game).filter((map) => map.canonical === server.map);
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
})
.map((server) => server.name)
.sort();
const coverage = includeHealth
? existingServers.map((name) => {
const operation = db()
.prepare(
"SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1"
)
.get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return {
name,
health: operation
? operation.state === 'queued'
? 'queued'
: 'starting'
: server?.container_state &&
['stopped', 'exited', 'created', 'dead'].includes(
server.container_state
)
? 'stopped'
: server
? serverObservation(server).health
: 'offline',
href: operation
? `/provisioning/${encodeURIComponent(operation.id)}`
: `/servers/${encodeURIComponent(name)}`
};
})
: [];
return {
existingServers,
coverage,
existing: existingServers.length,
missing: Math.max(0, entry.count - existingServers.length)
};
});
}
function fleetSnapshot() {
return housePartyTargets().map(name => ({ name,
server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
return housePartyTargets().map((name) => ({
name,
server:
db()
.prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?')
.get(name) ?? null,
creation:
db()
.prepare(
"SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1"
)
.get(name) ?? null,
removal:
db()
.prepare(
"SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')"
)
.get(name) ?? null
}));
}
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, includeHealth = true) {
const counts = coverage(entries, includeHealth);
// Preserve deployed lobbies before queued ones, then the lowest lobby IDs.
const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers].sort((a, b) =>
Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true })
).slice(0, mode === 'exact' ? entries[index].count : undefined));
const retainedServers = counts.flatMap((entry, index) =>
[...entry.existingServers]
.sort(
(a, b) =>
Number(!!getServer(b)) - Number(!!getServer(a)) ||
a.localeCompare(b, 'en', { numeric: true })
)
.slice(0, mode === 'exact' ? entries[index].count : undefined)
);
const retained = new Set(retainedServers);
const matching = new Set(counts.flatMap(entry => entry.existingServers));
const removals = mode === 'exact' ? housePartyTargets().filter(name => !retained.has(name)).map(name => ({ name,
reason: matching.has(name) ? 'Exceeds the target lobby count' : 'Outside this collection or already awaiting removal'
})) : [];
const matching = new Set(counts.flatMap((entry) => entry.existingServers));
const removals =
mode === 'exact'
? housePartyTargets()
.filter((name) => !retained.has(name))
.map((name) => ({
name,
reason: matching.has(name)
? 'Exceeds the target lobby count'
: 'Outside this collection or already awaiting removal'
}))
: [];
return { counts, retainedServers, removals };
}
function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] {
const occupied = db().prepare(`SELECT name FROM servers UNION
function allocate(
entries: BlueprintEntry[],
counts = coverage(entries)
): CollectionPreview['servers'] {
const occupied = db()
.prepare(
`SELECT name FROM servers UNION
SELECT server AS name FROM native_instances WHERE retired_at IS NULL UNION
SELECT server_id AS name FROM server_key_leases UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`)
OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`
)
.all() as { name: string }[];
const lobbies = new Map<string, Set<number>>();
for (const { name } of occupied) {
const match = /^(.*)_(\d+)$/.exec(name);
if (!match) continue;
const taken = lobbies.get(match[1]) ?? new Set<number>();
taken.add(Number(match[2])); lobbies.set(match[1], taken);
taken.add(Number(match[2]));
lobbies.set(match[1], taken);
}
const availableKeys = { t5: availableServerKeyCount('t5'), t6: availableServerKeyCount('t6') };
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const taken = lobbies.get(base) ?? new Set<number>();
const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null;
const name = lobby === null ? null : `${base}_${lobby}`;
const native = entry.recipe.native;
const port = null;
if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0;
if (hasSavedKey) availableKeys[entry.recipe.game]--;
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey };
}));
return entries.flatMap((entry, entryIndex) =>
Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const taken = lobbies.get(base) ?? new Set<number>();
const lobby =
Array.from({ length: 20 }, (_, index) => index + 1).find(
(number) => !taken.has(number)
) ?? null;
const name = lobby === null ? null : `${base}_${lobby}`;
const native = entry.recipe.native;
const port = null;
if (lobby !== null) {
taken.add(lobby);
lobbies.set(base, taken);
}
const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0;
if (hasSavedKey) availableKeys[entry.recipe.game]--;
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey };
})
);
}
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
function blockersFor(
servers: CollectionPreview['servers'],
entries: BlueprintEntry[] = []
): string[] {
const blockers: string[] = [];
for (const server of servers) if (server.name) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); }
for (const server of servers)
if (server.name) {
const owned = scalingOwnershipBlocker(server.name);
if (owned) blockers.push(owned);
}
if (!servers.length) return blockers;
if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.');
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
if (servers.some((server) => !server.native && !entries[server.entryIndex]?.recipe.native))
blockers.push('Select and prepare a host before launching.');
if (servers.some((server) => !server.name))
blockers.push(
'There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.'
);
return blockers;
}
@@ -116,11 +227,16 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
const blockers: string[] = [];
const native = (name: string) => {
if (nativeInstance(name)) return true;
const creation = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined;
const creation = db()
.prepare(
`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`
)
.get(name) as { id: string } | undefined;
return !!creation && nativeDeployment(creation.id);
};
if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.');
if (removals.some((server) => !native(server.name)))
blockers.push('Every server to remove must have a native host instance.');
for (const server of removals) {
const owned = scalingOwnershipBlocker(server.name);
if (owned) blockers.push(owned);
@@ -130,50 +246,118 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
return blockers;
}
export function previewCollection(blueprintId: unknown, mode: unknown = 'fill', selection?: Record<string, unknown>): CollectionPreview {
if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.');
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
export function previewCollection(
blueprintId: unknown,
mode: unknown = 'fill',
selection?: Record<string, unknown>
): CollectionPreview {
if (typeof blueprintId !== 'string' || !blueprintId)
throw new Error('Choose a collection to launch.');
if (mode !== 'fill' && mode !== 'exact')
throw new Error('Choose fill or exact collection mode.');
return db().transaction((): CollectionPreview => {
const { entries, blueprintName } = blueprintEntries(blueprintId);
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
for (const entry of entries) {
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
entry.recipe.native = nativeLaunchSelection(
{ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId },
entry.recipe.game
);
entry.recipe.image = entry.recipe.native.image!;
}
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
const servers = allocate(entries, counts);
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
const selections = servers.flatMap(server => server.native ? [server.native] : []);
const selections = servers.flatMap((server) => (server.native ? [server.native] : []));
const availability = selections.length ? launchAvailability(selections) : {};
const id = randomUUID();
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null);
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ...availability, ready: blockers.length === 0,
entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) };
db()
.prepare(
'DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL'
)
.run(new Date().toISOString());
db()
.prepare(
'INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)'
)
.run(
id,
expiresAt,
blueprintName,
JSON.stringify(entries),
JSON.stringify(servers),
mode,
mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null
);
return {
id,
expiresAt,
blueprintName,
mode,
retainedServers,
removals,
launchConcurrency: fleetSettings().createConcurrency,
servers,
blockers,
...availability,
ready: blockers.length === 0,
entries: entries.map((entry, index) => ({
count: entry.count,
...counts[index],
recipe: summarizeRecipe(entry.recipe)
}))
};
})();
}
/** Save all reviewed identities and keys atomically; native placement belongs to provisioning. */
export function queueCollection(input: { previewId?: unknown; keys?: unknown }): { ok: true; collectionId: string | null } {
if (typeof input.previewId !== 'string' || !input.previewId) throw new Error('Review a collection preview before launching.');
if (input.keys !== undefined && (!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))) throw new Error('Provide keys indexed by the exact server ID.');
export function queueCollection(input: { previewId?: unknown; keys?: unknown }): {
ok: true;
collectionId: string | null;
} {
if (typeof input.previewId !== 'string' || !input.previewId)
throw new Error('Review a collection preview before launching.');
if (
input.keys !== undefined &&
(!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))
)
throw new Error('Provide keys indexed by the exact server ID.');
const keys = (input.keys ?? {}) as Record<string, unknown>;
return db().transaction(() => {
const preview = db().prepare('SELECT * FROM server_collection_previews WHERE id = ?').get(input.previewId) as {
id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null; mode: CollectionMode; fleet_snapshot: string | null;
} | undefined;
if (!preview) throw new Error('Collection preview expired or was not found. Refresh the preview.');
if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id };
if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.');
const preview = db()
.prepare('SELECT * FROM server_collection_previews WHERE id = ?')
.get(input.previewId) as
| {
id: string;
expires_at: string;
blueprint_name: string;
entries: string;
servers: string;
collection_id: string | null;
mode: CollectionMode;
fleet_snapshot: string | null;
}
| undefined;
if (!preview)
throw new Error('Collection preview expired or was not found. Refresh the preview.');
if (preview.collection_id)
return { ok: true as const, collectionId: preview.collection_id };
if (preview.expires_at <= new Date().toISOString())
throw new Error('Collection preview expired. Refresh it before launching.');
const entries: BlueprintEntry[] = JSON.parse(preview.entries);
// Older previews embedded plugin bytes. Queued jobs need only the
// reviewed metadata; provisioning captures the current files at dispatch.
for (const entry of entries) entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file);
for (const entry of entries)
entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file);
const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers);
if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) {
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
if (
preview.mode === 'exact' &&
preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())
) {
throw new Error(
'The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.'
);
}
const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false);
const servers = reviewed;
@@ -181,7 +365,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
for (const server of servers) {
const recipe = entries[server.entryIndex].recipe;
const selection = server.native ?? recipe.native;
if (!selection) throw new Error('Select a native host before launching. Refresh the preview.');
if (!selection)
throw new Error('Select a native host before launching. Refresh the preview.');
const identity = JSON.stringify([recipe.game, selection]);
if (validatedSelections.has(identity)) continue;
// Recheck saved configuration once per selection without allocating
@@ -191,50 +376,102 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
}
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
if (blockers.length) throw new Error(blockers.join(' '));
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) {
throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.');
if (
entries.some(
(_entry, index) =>
counts[index].missing !==
reviewed.filter((server) => server.entryIndex === index).length
)
) {
throw new Error(
'Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.'
);
}
const expected = new Set(servers.map(server => server.name));
const expected = new Set(servers.map((server) => server.name));
for (const [name, value] of Object.entries(keys)) {
if (!expected.has(name) || typeof value !== 'string') throw new Error('Keys must be strings for server IDs shown in this preview.');
if (!expected.has(name) || typeof value !== 'string')
throw new Error('Keys must be strings for server IDs shown in this preview.');
}
if (!servers.length && !removals.length) return { ok: true as const, collectionId: null };
const collectionId = randomUUID();
db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)')
.run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers));
db()
.prepare(
'INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)'
)
.run(
collectionId,
preview.blueprint_name,
new Date().toISOString(),
preview.mode,
JSON.stringify(retainedServers)
);
removals.forEach((server, position) => {
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
const queued = db()
.prepare(
"SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'"
)
.get(server.name) as { id: string } | undefined;
let id: string;
if (queued && !getServer(server.name)) {
// No container or files exist yet: cancel the reservation instead of booting just to remove it.
const result = destroyServer(server.name, 'blueprint-exact');
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
if (!result.deploymentId)
throw new Error('Could not queue removal for ' + server.name);
id = result.deploymentId;
db().prepare("UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?").run(new Date().toISOString(), queued.id);
recordDeploymentEvent(queued.id, 'Queued creation cancelled by exact collection; saved key retained.');
db()
.prepare(
"UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?"
)
.run(new Date().toISOString(), queued.id);
recordDeploymentEvent(
queued.id,
'Queued creation cancelled by exact collection; saved key retained.'
);
deploymentStage(id, 'Cancelled queued creation; no server was started');
finishDeployment(id, { ok: true });
} else {
const result = destroyServer(server.name, 'blueprint-exact');
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
if (!result.deploymentId)
throw new Error('Could not queue removal for ' + server.name);
id = result.deploymentId;
}
db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);
db()
.prepare(
'INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)'
)
.run(collectionId, id, position);
});
servers.forEach((server, index) => {
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.');
const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name });
const recipe = {
...entries[server.entryIndex].recipe,
...(server.native ? { native: server.native } : {})
};
if (!recipe.native)
throw new Error('Select a native host before launching. Refresh the preview.');
const result = queueNativeCreationIntent({
server: server.name!,
game: recipe.game,
mapSlug: recipe.mapSlug,
selection: recipe.native,
recipe,
plutoKey: keys[server.name!] as string | undefined,
createdBy: 'ui',
blueprintName: preview.blueprint_name
});
// Throw so the outer transaction also reverses all earlier keys and children.
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
if (!result.ok)
throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db()
.prepare(
'INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)'
)
.run(collectionId, result.deploymentId, index);
});
db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id);
db()
.prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?')
.run(collectionId, preview.id);
return { ok: true as const, collectionId };
})();
}
@@ -243,39 +480,60 @@ const membershipSql = `SELECT collection_id, deployment_id, position FROM server
UNION ALL SELECT collection_id, deployment_id, position FROM server_collection_removals`;
function collectionCancellationCandidates(id: string) {
const creations = (db().prepare(`SELECT d.id FROM server_collection_members m
const creations = (
db()
.prepare(
`SELECT d.id FROM server_collection_members m
JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.collection_id=?
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.creation_id=d.id)`)
.all(id) as { id: string }[]).filter(job => canCancelCreation(job.id));
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.creation_id=d.id)`
)
.all(id) as { id: string }[]
).filter((job) => canCancelCreation(job.id));
// Exact launches can reuse manual removals or creation cleanup jobs. Membership
// alone must never give a launch permission to withdraw those operations.
const removals = (db().prepare(`SELECT d.id FROM server_collection_removals m
const removals = (
db()
.prepare(
`SELECT d.id FROM server_collection_removals m
JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.collection_id=? AND d.triggered_by='blueprint-exact'
AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.operation_id IS NOT NULL)
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.cleanup_id=d.id)`)
.all(id) as { id: string }[]).filter(job => canCancelRemoval(job.id));
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.cleanup_id=d.id)`
)
.all(id) as { id: string }[]
).filter((job) => canCancelRemoval(job.id));
return { creations, removals };
}
/** Withdraw the remaining launch atomically; partial instances keep durable cleanup jobs. */
export function cancelCollection(id: string): boolean {
return db().transaction(() => {
const collection = db().prepare('SELECT cancelled_at FROM server_launch_collections WHERE id=?').get(id) as { cancelled_at: string | null } | undefined;
const collection = db()
.prepare('SELECT cancelled_at FROM server_launch_collections WHERE id=?')
.get(id) as { cancelled_at: string | null } | undefined;
if (!collection) return false;
if (collection.cancelled_at) return true;
const { creations, removals } = collectionCancellationCandidates(id);
if (!creations.length && !removals.length) return false;
db().prepare('UPDATE server_launch_collections SET cancelled_at=? WHERE id=?').run(new Date().toISOString(), id);
db()
.prepare('UPDATE server_launch_collections SET cancelled_at=? WHERE id=?')
.run(new Date().toISOString(), id);
for (const creation of creations) {
if (!cancelCreation(creation.id, 'collection-cancel')) throw new Error('Could not cancel a collection creation. Refresh the launch and try again.');
if (!cancelCreation(creation.id, 'collection-cancel'))
throw new Error(
'Could not cancel a collection creation. Refresh the launch and try again.'
);
}
for (const removal of removals) {
// A shared removal remains requested until every owning launch withdraws.
const shared = db().prepare(`SELECT 1 FROM server_collection_removals m
const shared = db()
.prepare(
`SELECT 1 FROM server_collection_removals m
JOIN server_launch_collections c ON c.id=m.collection_id
WHERE m.deployment_id=? AND c.id!=? AND c.cancelled_at IS NULL LIMIT 1`).get(removal.id, id);
WHERE m.deployment_id=? AND c.id!=? AND c.cancelled_at IS NULL LIMIT 1`
)
.get(removal.id, id);
if (!shared) cancelRemoval(removal.id);
}
return true;
@@ -299,26 +557,59 @@ const summarySql = `SELECT c.id, c.name, c.created_at, c.cancelled_at, c.mode, C
LEFT JOIN server_deployments cleanup ON cleanup.id = cc.cleanup_id`;
export function listCollections(): CollectionSummary[] {
return db().prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC').all() as CollectionSummary[];
return db()
.prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC')
.all() as CollectionSummary[];
}
export function getCollection(id: string): CollectionDetail | null {
const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined;
const collection = db()
.prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id')
.get(id) as CollectionSummary | undefined;
if (!collection) return null;
const servers = db().prepare(`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
const servers = db()
.prepare(
`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
(d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers'];
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`
)
.all(id) as CollectionDetail['servers'];
const queue = creationQueue();
const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1]));
const stored = db().prepare('SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string; retained_server_deletions: string };
const cancellations = db().prepare(`SELECT cc.creation_id, cc.cleanup_id AS cleanupId, d.state, d.stage, d.error
const stored = db()
.prepare(
'SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?'
)
.get(id) as { retained_servers: string; retained_server_deletions: string };
const cancellations = db()
.prepare(
`SELECT cc.creation_id, cc.cleanup_id AS cleanupId, d.state, d.stage, d.error
FROM server_collection_members m JOIN native_creation_cancellations cc ON cc.creation_id=m.deployment_id
JOIN server_deployments d ON d.id=cc.cleanup_id WHERE m.collection_id=?`).all(id) as
(NonNullable<CollectionDetail['servers'][number]['cancellation']> & { creation_id: string })[];
const cleanupByCreation = new Map(cancellations.map(({ creation_id, ...cleanup }) => [creation_id, cleanup]));
JOIN server_deployments d ON d.id=cc.cleanup_id WHERE m.collection_id=?`
)
.all(id) as (NonNullable<CollectionDetail['servers'][number]['cancellation']> & {
creation_id: string;
})[];
const cleanupByCreation = new Map(
cancellations.map(({ creation_id, ...cleanup }) => [creation_id, cleanup])
);
const candidates = collection.cancelled_at ? null : collectionCancellationCandidates(id);
return { collection, canCancel: !!candidates && (candidates.creations.length > 0 || candidates.removals.length > 0),
retainedServers: JSON.parse(stored.retained_servers), retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length },
servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null, cancellation: cleanupByCreation.get(server.id) ?? null })) };
return {
collection,
canCancel:
!!candidates && (candidates.creations.length > 0 || candidates.removals.length > 0),
retainedServers: JSON.parse(stored.retained_servers),
retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions),
queue: {
concurrency: fleetSettings().createConcurrency,
active: queue.active,
queued: queue.queued.length
},
servers: servers.map((server) => ({
...server,
queuePosition: positions.get(server.id) ?? null,
cancellation: cleanupByCreation.get(server.id) ?? null
}))
};
}
src/lib/server/fleet/config-inspect.ts
+72 -19
View File
@@ -1,62 +1,115 @@
import { composeServerCfg, parseCfg } from './config-compose.js';
import { selectedZombieMap, mapRotation } from './map-aliases.js';
import { configWarnings, type ConfigAssignment, type ConfigInspection, type ConfigSource } from '../../config-workbench.js';
import {
configWarnings,
type ConfigAssignment,
type ConfigInspection,
type ConfigSource
} from '../../config-workbench.js';
export interface ConfigLayer { source: ConfigSource; text: string; }
export interface ConfigLayer {
source: ConfigSource;
text: string;
}
/** Trace file composition, including assignments left in place by the composer.
* This is an intended-file preview, not a query of live runtime dvars. */
export function inspectConfig(baseline: ConfigLayer, overlays: ConfigLayer[], game: string | null, map: string | null, mapSlug?: string | null): ConfigInspection {
export function inspectConfig(
baseline: ConfigLayer,
overlays: ConfigLayer[],
game: string | null,
map: string | null,
mapSlug?: string | null
): ConfigInspection {
const sorted = [...overlays].sort((a, b) => a.source.id! - b.source.id!);
const parsed = parseCfg(baseline.text);
type TracedLine = { line: string; history: ConfigAssignment[] };
function assignment(line: string, source: ConfigSource, index: number): ConfigAssignment {
const match = /^\s*(?:(?:set|seta)\s+)(\S+)\s+(.*)$/i.exec(line)
?? /^\s*(sv_maprotation)\s+(.*)$/i.exec(line);
const match =
/^\s*(?:(?:set|seta)\s+)(\S+)\s+(.*)$/i.exec(line) ??
/^\s*(sv_maprotation)\s+(.*)$/i.exec(line);
return { value: match?.[2] ?? '', source, line: index + 1 };
}
const lines: TracedLine[] = parsed.order.map((entry, index) => ({
line: entry.line, history: [assignment(entry.line, baseline.source, index)]
line: entry.line,
history: [assignment(entry.line, baseline.source, index)]
}));
const replacements = new Map<string, TracedLine>();
const warnings = configWarnings(baseline.text, false).map(w => `${baseline.source.name}: ${w}`);
const warnings = configWarnings(baseline.text, false).map(
(w) => `${baseline.source.name}: ${w}`
);
for (const overlay of sorted) {
warnings.push(...configWarnings(overlay.text, true).map(w => `${overlay.source.name}: ${w}`));
warnings.push(
...configWarnings(overlay.text, true).map((w) => `${overlay.source.name}: ${w}`)
);
overlay.text.split(/\r?\n/).forEach((line, index) => {
const match = /^\s*set\s+(\S+)\s+/.exec(line);
if (!match) return;
const previous = replacements.get(match[1]);
replacements.set(match[1], { line, history: [...(previous?.history ?? []), assignment(line, overlay.source, index)] });
replacements.set(match[1], {
line,
history: [...(previous?.history ?? []), assignment(line, overlay.source, index)]
});
});
}
for (const [key, replacement] of replacements) {
const index = parsed.entries.get(key);
if (index === undefined) lines.push(replacement);
else lines[index] = { line: replacement.line, history: [...lines[index].history, ...replacement.history] };
else
lines[index] = {
line: replacement.line,
history: [...lines[index].history, ...replacement.history]
};
}
const sources = [baseline.source, ...sorted.map(o => o.source)];
const sources = [baseline.source, ...sorted.map((o) => o.source)];
const selected = selectedZombieMap(game, map, mapSlug);
if (selected) {
const source: ConfigSource = { id: null, name: 'Server map selection', version: null, kind: 'map' };
const source: ConfigSource = {
id: null,
name: 'Server map selection',
version: null,
kind: 'map'
};
sources.push(source);
const rotation = /^\s*(?:(?:set|seta)\s+)?sv_maprotation\s+/i;
const history = lines.filter(entry => rotation.test(entry.line)).flatMap(entry => entry.history);
for (let i = lines.length - 1; i >= 0; i--) if (rotation.test(lines[i].line)) lines.splice(i, 1);
const history = lines
.filter((entry) => rotation.test(entry.line))
.flatMap((entry) => entry.history);
for (let i = lines.length - 1; i >= 0; i--)
if (rotation.test(lines[i].line)) lines.splice(i, 1);
const line = `set sv_maprotation "${mapRotation(selected)}"`;
lines.push({ line, history: [...history, assignment(line, source, 0)] });
}
const byKey = new Map<string, { key: string; history: ConfigAssignment[] }>();
for (const entry of lines) {
const match = /^\s*(?:set|seta)\s+(\S+)\s+/i.exec(entry.line) ?? /^\s*(sv_maprotation)\s+/i.exec(entry.line);
const match =
/^\s*(?:set|seta)\s+(\S+)\s+/i.exec(entry.line) ??
/^\s*(sv_maprotation)\s+/i.exec(entry.line);
if (!match) continue;
const normalized = match[1].toLowerCase();
const existing = byKey.get(normalized);
byKey.set(normalized, { key: match[1], history: [...(existing?.history ?? []), ...entry.history] });
byKey.set(normalized, {
key: match[1],
history: [...(existing?.history ?? []), ...entry.history]
});
}
return {
text: composeServerCfg(baseline.text, sorted.map(o => ({ id: o.source.id!, text: o.text })), game, map, mapSlug),
sources, warnings,
settings: [...byKey.values()].map(({ key, history }) => ({ key, history, value: history.at(-1)!.value, source: history.at(-1)!.source }))
text: composeServerCfg(
baseline.text,
sorted.map((o) => ({ id: o.source.id!, text: o.text })),
game,
map,
mapSlug
),
sources,
warnings,
settings: [...byKey.values()]
.map(({ key, history }) => ({
key,
history,
value: history.at(-1)!.value,
source: history.at(-1)!.source
}))
.sort((a, b) => a.key.localeCompare(b.key))
};
}
src/lib/server/fleet/drift.ts
+360 -114
View File
@@ -1,6 +1,12 @@
import { putFileBlob } from '../db/file-blobs.js';
import { getVersionContent } from './scripts.js';
import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js';
import {
readdir,
readFile,
stat,
nativeFileManifest,
type FileFingerprint
} from '../backends/files.js';
import { randomUUID } from 'node:crypto';
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
import { join, posix, relative, sep } from 'node:path';
@@ -27,12 +33,18 @@ async function scanBatch<T, R>(items: readonly T[], work: (item: T) => Promise<R
async function pump() {
while (!failed && cursor < items.length) {
const index = cursor++;
try { results[index] = await work(items[index]); }
catch (error) { failed = true; throw error; }
try {
results[index] = await work(items[index]);
} catch (error) {
failed = true;
throw error;
}
}
}
const workers = await Promise.allSettled(Array.from({ length: Math.min(4, items.length) }, pump));
const failure = workers.find(result => result.status === 'rejected');
const workers = await Promise.allSettled(
Array.from({ length: Math.min(4, items.length) }, pump)
);
const failure = workers.find((result) => result.status === 'rejected');
if (failure?.status === 'rejected') throw failure.reason;
return results;
}
@@ -73,15 +85,22 @@ function toPosix(p: string): string {
}
function hasActiveBeta(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
return !!db()
.prepare(
`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`
)
.get(server);
}
async function requireServerDirectory(server: string): Promise<void> {
try {
if (!(await stat(serverDir(server))).isDirectory()) throw new Error('path is not a directory');
if (!(await stat(serverDir(server))).isDirectory())
throw new Error('path is not a directory');
} catch (error) {
throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`);
throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`, {
cause: error
});
}
}
@@ -92,7 +111,10 @@ async function readExistingFile(path: string): Promise<Buffer | null> {
});
}
async function fingerprint(path: string, manifest: Map<string, FileFingerprint> | null): Promise<FileFingerprint | null> {
async function fingerprint(
path: string,
manifest: Map<string, FileFingerprint> | null
): Promise<FileFingerprint | null> {
if (manifest) return manifest.get(path) ?? null;
const content = await readExistingFile(path);
return content === null ? null : { size: content.byteLength, sha256: sha256Buffer(content) };
@@ -102,10 +124,11 @@ async function walkFiles(root: string): Promise<string[]> {
const out: string[] = [];
let directories = [root];
while (directories.length) {
const children = await scanBatch(directories, async dir => {
const children = await scanBatch(directories, async (dir) => {
let entries;
try { entries = await readdir(dir, { withFileTypes: true }); }
catch (error) {
try {
entries = await readdir(dir, { withFileTypes: true });
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [];
throw error;
}
@@ -131,51 +154,83 @@ async function walkFiles(root: string): Promise<string[]> {
* artifacts with the promoted versions.
*/
export async function scanServerDrift(server: string): Promise<DriftEntry[]> {
return trackActivity(`drift:${server}`, `Checking files on ${server}`, () => scanServerFiles(server));
return trackActivity(`drift:${server}`, `Checking files on ${server}`, () =>
scanServerFiles(server)
);
}
async function scanServerFiles(server: string): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv || srv.destroyed_at || hasActiveBeta(server)) return [];
// A missing mount is unavailable evidence, not an empty managed tree.
const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) => join(destOfKind(server, kind), script.target_rel_path));
if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id) expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server)));
if (srv.created_by && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE));
const manifest = await nativeFileManifest(serverDir(server), DRIFT_DESTS.map(kind => destOfKind(server, kind)), [...DRIFT_IGNORE_FILENAMES], expectedPaths);
const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) =>
join(destOfKind(server, kind), script.target_rel_path)
);
if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id)
expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server)));
if (
srv.created_by &&
srv.game_mode === 'zombie' &&
getServerMapConfig(srv.game, srv.map, srv.map_slug)
)
expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE));
const manifest = await nativeFileManifest(
serverDir(server),
DRIFT_DESTS.map((kind) => destOfKind(server, kind)),
[...DRIFT_IGNORE_FILENAMES],
expectedPaths
);
if (!manifest) await requireServerDirectory(server);
const drift = await scanManagedForServer(server, manifest);
// Previous inventory is an audit of installed bytes, not permission to
// retain artifacts whose scope was moved or removed from the library.
const managed = new Set(promotedArtifactMetadata(srv.game, srv.map)
.map(({ kind, script }) => `${kind}:${script.target_rel_path}`));
const managed = new Set(
promotedArtifactMetadata(srv.game, srv.map).map(
({ kind, script }) => `${kind}:${script.target_rel_path}`
)
);
const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined;
if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`);
// Without a promoted baseline, Plutainer owns the seeded config. Keep it
// until a baseline is promoted; startup arguments handle map selection.
const keepDefaultConfig = srv?.game
&& selectedZombieMap(srv.game, srv.map, srv.map_slug)
&& !baseline?.current_version_id;
const keepDefaultConfig =
srv?.game &&
selectedZombieMap(srv.game, srv.map, srv.map_slug) &&
!baseline?.current_version_id;
// Scan for untracked files only after resolving managed targets.
for (const kind of DRIFT_DESTS) {
const dest = destOfKind(server, kind);
const onDisk = manifest
? [...manifest.keys()].filter(path => path.startsWith(`${dest}${sep}`)).map(path => toPosix(relative(dest, path)))
.filter(path => !path.split('/').some(name => DRIFT_IGNORE_FILENAMES.has(name))).sort()
? [...manifest.keys()]
.filter((path) => path.startsWith(`${dest}${sep}`))
.map((path) => toPosix(relative(dest, path)))
.filter(
(path) => !path.split('/').some((name) => DRIFT_IGNORE_FILENAMES.has(name))
)
.sort()
: await walkFiles(dest);
if (onDisk.length === 0) continue;
const untracked = await scanBatch(onDisk, async (relPath): Promise<DriftEntry | null> => {
if (managed.has(`${kind}:${relPath}`)) return null;
if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) return null;
if (kind === 'config' && relPath === MAP_CONFIG_FILE && srv?.created_by
&& srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) return null;
if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server))
return null;
if (
kind === 'config' &&
relPath === MAP_CONFIG_FILE &&
srv?.created_by &&
srv.game_mode === 'zombie' &&
getServerMapConfig(srv.game, srv.map, srv.map_slug)
)
return null;
const abs = join(dest, relPath);
try {
const info = await fingerprint(abs, manifest);
if (!info) return null;
return {
server, relPath,
server,
relPath,
...info,
kind,
driftKind: 'untracked'
@@ -196,38 +251,63 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
// Every promoted artifact whose scope matches this server's game/map should
// have its target file present on disk. Anything absent is 'missing' drift.
// Configs are handled as a composed layer (baseline + overlays → one cfg).
async function scanManagedForServer(server: string, manifest: Map<string, FileFingerprint> | null): Promise<DriftEntry[]> {
async function scanManagedForServer(
server: string,
manifest: Map<string, FileFingerprint> | null
): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv) return [];
const out: DriftEntry[] = [];
// Provisioned servers load this generated file after their main cfg. It
// remains managed even when there is no promoted baseline yet.
const mapConfig = srv.created_by && srv.game_mode === 'zombie'
? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null;
const mapConfig =
srv.created_by && srv.game_mode === 'zombie'
? getServerMapConfig(srv.game, srv.map, srv.map_slug)
: null;
if (mapConfig) {
const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE);
const content = await fingerprint(abs, manifest);
if (content?.sha256 !== sha256Buffer(mapConfig)) {
out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing',
size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) });
out.push({
server,
relPath: MAP_CONFIG_FILE,
kind: 'config',
driftKind: content ? 'replace' : 'missing',
size: mapConfig.byteLength,
sha256: sha256Buffer(mapConfig)
});
}
}
// Inventory records the last managed write, not the current disk bytes.
// Check both so edits and outdated versions are detected on tracked files.
const managed = await scanBatch(promotedArtifactMetadata(srv.game, srv.map), async ({ kind, script, version }): Promise<DriftEntry | null> => {
const content = await fingerprint(join(destOfKind(server, kind), script.target_rel_path), manifest);
const inventory = getServerScript(server, script.target_rel_path);
if (content?.sha256 === version.sha256 && inventory?.sha256 === version.sha256
&& inventory.script_version_id === version.id) return null;
return {
server, relPath: script.target_rel_path, kind,
driftKind: content ? 'replace' : 'missing',
size: version.size, sha256: version.sha256,
scriptId: script.id, scriptVersionId: version.id
};
});
const managed = await scanBatch(
promotedArtifactMetadata(srv.game, srv.map),
async ({ kind, script, version }): Promise<DriftEntry | null> => {
const content = await fingerprint(
join(destOfKind(server, kind), script.target_rel_path),
manifest
);
const inventory = getServerScript(server, script.target_rel_path);
if (
content?.sha256 === version.sha256 &&
inventory?.sha256 === version.sha256 &&
inventory.script_version_id === version.id
)
return null;
return {
server,
relPath: script.target_rel_path,
kind,
driftKind: content ? 'replace' : 'missing',
size: version.size,
sha256: version.sha256,
scriptId: script.id,
scriptVersionId: version.id
};
}
);
out.push(...managed.filter((entry): entry is DriftEntry => entry !== null));
// Composed cfg: the effective dedicated_sp.cfg is baseline + all promoted
@@ -247,13 +327,24 @@ async function scanManagedForServer(server: string, manifest: Map<string, FileFi
// Older launches tracked the promoted bytes before Plutainer
// applied the instance password. Reconcile only that known
// before-image, and only after verifying the full effective file.
if (composed.sha256 !== composed.promotedSha256 && content?.sha256 === composed.sha256
&& inv?.sha256 === composed.promotedSha256 && !hasActiveBeta(server)) {
recordScript(server, targetRel, inv.script_version_id, composed.sha256,
composed.content.byteLength, 'runtime-config-reconciliation');
if (
composed.sha256 !== composed.promotedSha256 &&
content?.sha256 === composed.sha256 &&
inv?.sha256 === composed.promotedSha256 &&
!hasActiveBeta(server)
) {
recordScript(
server,
targetRel,
inv.script_version_id,
composed.sha256,
composed.content.byteLength,
'runtime-config-reconciliation'
);
inv = getServerScript(server, targetRel);
}
const onDiskMatches = content?.sha256 === composed.sha256 && inv?.sha256 === composed.sha256;
const onDiskMatches =
content?.sha256 === composed.sha256 && inv?.sha256 === composed.sha256;
if (!onDiskMatches) {
out.push({
server,
@@ -282,7 +373,8 @@ async function resolveManagedEntry(server: string, entry: DriftEntry): Promise<B
if (entry.kind === 'config') {
const srv = getServer(server);
if (!srv?.game) return null;
if (entry.relPath === MAP_CONFIG_FILE) return getServerMapConfig(srv.game, srv.map, srv.map_slug);
if (entry.relPath === MAP_CONFIG_FILE)
return getServerMapConfig(srv.game, srv.map, srv.map_slug);
const composed = composeExpectedCfg(server, srv.game, srv.map, srv.map_slug);
return composed?.content ?? null;
}
@@ -299,7 +391,9 @@ function composeExpectedCfg(
): { content: Buffer; sha256: string; promotedSha256: string } | null {
const promoted = getPromotedServerConfig(game, map, mapSlug);
if (!promoted) return null;
const content = Buffer.from(applyRuntimePassword(promoted.toString('utf8'), runtimePasswordForServer(server)));
const content = Buffer.from(
applyRuntimePassword(promoted.toString('utf8'), runtimePasswordForServer(server))
);
return { content, sha256: sha256Buffer(content), promotedSha256: sha256Buffer(promoted) };
}
@@ -318,46 +412,70 @@ export async function scanFleetDrift(): Promise<DriftEntry[]> {
}
export interface RunDriftInput {
servers?: string[]; // if omitted, scans the whole fleet
servers?: string[]; // if omitted, scans the whole fleet
dryRun?: boolean;
triggeredBy?: 'ui' | 'sweep' | 'cron';
expectedFingerprint?: string;
}
export class DriftPlanChanged extends Error {
constructor(public plan: DriftPlan) { super('The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'); }
constructor(public plan: DriftPlan) {
super(
'The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'
);
}
}
function driftTargets(servers?: string[]): string[] {
const registered = listServers().map(server => server.name);
if (servers && (!servers.length || servers.some(name => !registered.includes(name)))) throw new Error('Select at least one registered server. Refresh the fleet if a server was removed.');
const registered = listServers().map((server) => server.name);
if (servers && (!servers.length || servers.some((name) => !registered.includes(name))))
throw new Error(
'Select at least one registered server. Refresh the fleet if a server was removed.'
);
return [...new Set(servers ?? registered)].sort();
}
async function describePlan(servers: string[], entries: DriftEntry[]): Promise<DriftPlan> {
const files: DriftPlanFile[] = [];
for (const entry of entries) {
const before = await readFile(join(destOfKind(entry.server, entry.kind), entry.relPath)).catch((error: NodeJS.ErrnoException) => {
const before = await readFile(
join(destOfKind(entry.server, entry.kind), entry.relPath)
).catch((error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return null;
throw error;
});
const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry);
files.push({ server: entry.server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null });
const after =
entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry);
files.push({
server: entry.server,
relPath: entry.relPath,
kind: entry.kind,
driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null,
afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null,
afterSize: after?.byteLength ?? null
});
}
return buildPlan(servers, files);
}
function buildPlan(servers: string[], files: DriftPlanFile[]): DriftPlan {
files.sort((a, b) => `${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`));
return { servers, files, scannedAt: new Date().toISOString(), fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files }))) };
files.sort((a, b) =>
`${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`)
);
return {
servers,
files,
scannedAt: new Date().toISOString(),
fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files })))
};
}
export async function previewDrift(servers?: string[]): Promise<DriftPlan> {
const targets = driftTargets(servers);
const entries: DriftEntry[] = [];
for (const server of targets) entries.push(...await scanServerDrift(server));
for (const server of targets) entries.push(...(await scanServerDrift(server)));
return describePlan(targets, entries);
}
@@ -373,7 +491,10 @@ export interface RunDriftResult {
}
function deploymentIdNow(): string {
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
const stamp = new Date()
.toISOString()
.replace(/[-:]/g, '')
.replace(/\.\d+Z$/, 'Z');
return `dep-${stamp}-drift-${randomUUID().slice(0, 8)}`;
}
@@ -386,47 +507,103 @@ export interface PreparedDriftCampaign {
export async function prepareDriftCampaign(server: string): Promise<PreparedDriftCampaign> {
const servers = driftTargets([server]);
const entries = await scanServerDrift(server);
const files = await scanBatch(entries, async entry => {
const files = await scanBatch(entries, async (entry) => {
const before = await readExistingFile(join(destOfKind(server, entry.kind), entry.relPath));
const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry);
if (entry.driftKind !== 'untracked' && !after) throw new Error(`Cannot resolve promoted content for ${entry.relPath}`);
return { before, after, file: { server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } };
const after =
entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry);
if (entry.driftKind !== 'untracked' && !after)
throw new Error(`Cannot resolve promoted content for ${entry.relPath}`);
return {
before,
after,
file: {
server,
relPath: entry.relPath,
kind: entry.kind,
driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null,
afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null,
afterSize: after?.byteLength ?? null
}
};
});
if (entries.length) await requireServerDirectory(server);
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
return { plan: buildPlan(servers, files.map(({ file }) => file)), files };
if (hasActiveBeta(server))
throw new Error('Active beta owns this server; reconciliation deferred');
return {
plan: buildPlan(
servers,
files.map(({ file }) => file)
),
files
};
}
/** Record the audited plan atomically with a queued rebuild, never a second restart. */
export function createDriftRebuildCampaign(
server: string, operationId: string, incidentId: string | number, prepared: PreparedDriftCampaign
server: string,
operationId: string,
incidentId: string | number,
prepared: PreparedDriftCampaign
): string | null {
if (!prepared.files.length) return null;
if (prepared.plan.servers.length !== 1 || prepared.plan.servers[0] !== server
|| prepared.files.some(({ file }) => file.server !== server)) throw new Error('Drift evidence belongs to another server');
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
if (
prepared.plan.servers.length !== 1 ||
prepared.plan.servers[0] !== server ||
prepared.files.some(({ file }) => file.server !== server)
)
throw new Error('Drift evidence belongs to another server');
if (hasActiveBeta(server))
throw new Error('Active beta owns this server; reconciliation deferred');
return db().transaction(() => {
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'").get(operationId, server)) {
if (
!db()
.prepare(
"SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'"
)
.get(operationId, server)
) {
throw new Error('Drift campaign requires the matching rebuild operation');
}
const id = deploymentIdNow();
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')")
db()
.prepare(
"INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')"
)
.run(id, new Date().toISOString());
const insertFile = db().prepare(`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
for (const { file, before, after } of prepared.files) {
insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated',
file.relPath, file.beforeSha256, file.afterSha256, putFileBlob(before), putFileBlob(after));
insertFile.run(
id,
server,
after === null ? 'removed' : before === null ? 'added' : 'updated',
file.relPath,
file.beforeSha256,
file.afterSha256,
putFileBlob(before),
putFileBlob(after)
);
}
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId);
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)")
db()
.prepare(
`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
VALUES (?, ?, ?, 'pending', ?)`
)
.run(id, server, prepared.files.length, operationId);
db()
.prepare(
"INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)"
)
.run(operationId, id, server);
db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(id, `Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`);
db()
.prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(
id,
`Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`
);
emit({ type: 'deployment-opened', deploymentId: id, action: 'drift', servers: [server] });
return id;
})();
@@ -455,7 +632,12 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
}
// Recheck identities, existing bytes, and replacement bytes before any write.
const plan = input.expectedFingerprint ? await describePlan(scanTargets, perServer.flatMap(server => server.entries)) : null;
const plan = input.expectedFingerprint
? await describePlan(
scanTargets,
perServer.flatMap((server) => server.entries)
)
: null;
if (plan && plan.fingerprint !== input.expectedFingerprint) throw new DriftPlanChanged(plan);
const totalDrift = perServer.reduce((sum, s) => sum + s.entries.length, 0);
if (totalDrift === 0 || dryRun) {
@@ -464,7 +646,9 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
perServer: perServer.map((s) => ({
server: s.server,
removed: s.entries.filter((e) => e.driftKind === 'untracked').map((e) => e.relPath),
restored: s.entries.filter((e) => e.driftKind !== 'untracked').map((e) => e.relPath),
restored: s.entries
.filter((e) => e.driftKind !== 'untracked')
.map((e) => e.relPath),
skipped: []
})),
totalRemoved: perServer.reduce(
@@ -484,9 +668,11 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
// can be inserted as we go.
const depId = deploymentIdNow();
const now = new Date().toISOString();
db().prepare(
`INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)`
).run(depId, now, triggeredBy);
db()
.prepare(
`INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)`
)
.run(depId, now, triggeredBy);
const insertServer = db().prepare(
`INSERT INTO deployment_servers
(deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error)
@@ -511,53 +697,107 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
try {
await requireServerDirectory(s.server);
if (hasActiveBeta(s.server)) {
s.skipped.push(`${entry.relPath}: active beta owns this server; reconciliation deferred`);
s.skipped.push(
`${entry.relPath}: active beta owns this server; reconciliation deferred`
);
continue;
}
const reviewed = plan?.files.find(file => file.server === s.server && file.kind === entry.kind && file.relPath === entry.relPath);
const reviewed = plan?.files.find(
(file) =>
file.server === s.server &&
file.kind === entry.kind &&
file.relPath === entry.relPath
);
if (reviewed) {
const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; });
const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return null;
throw error;
});
if ((current ? sha256Buffer(current) : null) !== reviewed.beforeSha256) {
s.skipped.push(`${entry.relPath}: changed after review; scan and review again`);
s.skipped.push(
`${entry.relPath}: changed after review; scan and review again`
);
continue;
}
}
if (entry.driftKind === 'untracked') {
// Capture the reviewed deletion without touching the live tree.
const content = await readFile(abs);
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
if (hasActiveBeta(s.server))
throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, putFileBlob(content), null);
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) });
const file = insertFile.run(
depId,
s.server,
'removed',
entry.relPath,
sha256Buffer(content),
null,
putFileBlob(content),
null
);
queueFileChange({
sourceKind: 'campaign',
sourceId: depId,
server: s.server,
kind: entry.kind,
relPath: entry.relPath,
content: null,
inventorySource: `drift:${depId}`,
deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content)
});
})();
s.removed.push(entry.relPath);
} else {
// Persist exact replacement bytes so waiting cannot change the plan.
const written = await resolveManagedEntry(s.server, entry);
if (!written) {
s.skipped.push(`${entry.relPath}: missing artifact could not be resolved from the DB`);
s.skipped.push(
`${entry.relPath}: missing artifact could not be resolved from the DB`
);
continue;
}
if (reviewed && sha256Buffer(written) !== reviewed.afterSha256) {
s.skipped.push(`${entry.relPath}: promoted content changed after review; scan and review again`);
s.skipped.push(
`${entry.relPath}: promoted content changed after review; scan and review again`
);
continue;
}
const before = await readFile(abs).catch((err: NodeJS.ErrnoException) => {
if (err.code === 'ENOENT') return null;
throw err;
});
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
if (hasActiveBeta(s.server))
throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath,
before ? sha256Buffer(before) : null, sha256Buffer(written), putFileBlob(before), putFileBlob(written));
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: written,
scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed ? reviewed.beforeSha256 : before ? sha256Buffer(before) : null });
const file = insertFile.run(
depId,
s.server,
before ? 'updated' : 'added',
entry.relPath,
before ? sha256Buffer(before) : null,
sha256Buffer(written),
putFileBlob(before),
putFileBlob(written)
);
queueFileChange({
sourceKind: 'campaign',
sourceId: depId,
server: s.server,
kind: entry.kind,
relPath: entry.relPath,
content: written,
scriptVersionId:
entry.kind === 'config' ? null : (entry.scriptVersionId ?? null),
inventorySource: `drift:${depId}`,
deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed
? reviewed.beforeSha256
: before
? sha256Buffer(before)
: null
});
})();
s.restored.push(entry.relPath);
}
@@ -579,7 +819,10 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
return {
deploymentId: null,
perServer: perServer.map((s) => ({
server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
server: s.server,
removed: s.removed,
restored: s.restored,
skipped: s.skipped
})),
totalRemoved: 0,
totalRestored: 0,
@@ -598,7 +841,10 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
return {
deploymentId: depId,
perServer: perServer.map((s) => ({
server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
server: s.server,
removed: s.removed,
restored: s.restored,
skipped: s.skipped
})),
totalRemoved: perServer.reduce((sum, s) => sum + s.removed.length, 0),
totalRestored: perServer.reduce((sum, s) => sum + s.restored.length, 0),
src/lib/server/fleet/health-observations.ts
+226 -69
View File
@@ -4,7 +4,11 @@ import { db } from '../db/index.js';
import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
import { nativeInstance, type NativeInstance } from '../backends/native.js';
import { nativeHealth } from '../backends/native.js';
import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js';
import {
rconStatusResult,
agentRconStatusResult,
type AgentRconReply
} from '../integrations/rcon.js';
import { withOperationSource } from '../hosts/operation-context.js';
import { getServer } from './servers.js';
import { latestGameSubmission } from './lifecycle-evidence.js';
@@ -14,8 +18,15 @@ import { ONLINE_MAX_MS } from '../../server-freshness.js';
const MAX_AGE_MS = 30_000;
const MAX_SERVERS = 1024;
type Probe = 'inspect' | 'rcon';
interface Observation<T> { result: T; observedAt: string }
interface Identity { key: string; port: number | null; checkedAt: string | null }
interface Observation<T> {
result: T;
observedAt: string;
}
interface Identity {
key: string;
port: number | null;
checkedAt: string | null;
}
interface Entry<T> {
identity: Identity;
startedAt: number;
@@ -28,21 +39,50 @@ export class MonitoringSkippedError extends Error {}
function identity(server: string, probe: Probe, snapshot?: MonitoringSnapshot): Identity {
const row = snapshot ? snapshot.get(server)?.server : getServer(server);
if (!row || row.destroyed_at) throw new MonitoringChangedError('Server is no longer registered; monitoring evidence is unavailable.');
const native = snapshot ? snapshot.get(server)?.native : row.native_instance_id ? nativeInstance(server) : null;
if (row.native_instance_id && (!native || (snapshot ? snapshot.get(server)?.host : getHost(native.host_id))?.connectionState !== 'online')) {
if (!row || row.destroyed_at)
throw new MonitoringChangedError(
'Server is no longer registered; monitoring evidence is unavailable.'
);
const native = snapshot
? snapshot.get(server)?.native
: row.native_instance_id
? nativeInstance(server)
: null;
if (
row.native_instance_id &&
(!native ||
(snapshot ? snapshot.get(server)?.host : getHost(native.host_id))?.connectionState !==
'online')
) {
throw new Error('Host agent is offline; monitoring evidence is unavailable.');
}
// Stop requests also invalidate observations, even though they do not
// establish a new game-telemetry epoch in latestGameSubmission().
const savedLifecycle = snapshot?.get(server)?.lifecycle;
const lifecycle = snapshot ? savedLifecycle ? { id: savedLifecycle.id, action: savedLifecycle.action, submitted_at: savedLifecycle.submitted_at } : null : db().prepare(`SELECT id, action, submitted_at FROM server_deployments
WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) ?? null;
const lifecycle = snapshot
? savedLifecycle
? {
id: savedLifecycle.id,
action: savedLifecycle.action,
submitted_at: savedLifecycle.submitted_at
}
: null
: (db()
.prepare(
`SELECT id, action, submitted_at FROM server_deployments
WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`
)
.get(server) ?? null);
return {
key: JSON.stringify([row.created_at, row.desired_state, snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server), lifecycle,
key: JSON.stringify([
row.created_at,
row.desired_state,
snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server),
lifecycle,
native ? [native.host_id, native.id, native.generation] : null,
probe === 'rcon' ? [row.rcon_host, config.rcon] : null]),
port: probe === 'rcon' ? native?.port ?? row.game_port : null,
probe === 'rcon' ? [row.rcon_host, config.rcon] : null
]),
port: probe === 'rcon' ? (native?.port ?? row.game_port) : null,
checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at
};
}
@@ -58,14 +98,32 @@ function reusable(startedAt: number): boolean {
return age >= 0 && age < MAX_AGE_MS;
}
async function observe<T extends { ok: boolean }>(server: string, fresh: boolean, probe: Probe,
entries: Map<string, Entry<T>>, request: (server: string) => Promise<T>, snapshot?: MonitoringSnapshot, batch = false): Promise<Observation<T>> {
async function observe<T extends { ok: boolean }>(
server: string,
fresh: boolean,
probe: Probe,
entries: Map<string, Entry<T>>,
request: (server: string) => Promise<T>,
snapshot?: MonitoringSnapshot,
batch = false
): Promise<Observation<T>> {
let current: Identity;
try { current = identity(server, probe, snapshot); }
catch (error) { entries.delete(server); throw error; }
try {
current = identity(server, probe, snapshot);
} catch (error) {
entries.delete(server);
throw error;
}
const saved = entries.get(server);
const savedAt = saved?.observation ? Date.parse(saved.observation.observedAt) : saved?.startedAt ?? 0;
if (!fresh && saved && sameIdentity(saved.identity, current) && !newerEvidence(current, savedAt)) {
const savedAt = saved?.observation
? Date.parse(saved.observation.observedAt)
: (saved?.startedAt ?? 0);
if (
!fresh &&
saved &&
sameIdentity(saved.identity, current) &&
!newerEvidence(current, savedAt)
) {
if (saved.pending) return saved.pending;
if (saved.observation && reusable(saved.startedAt)) return saved.observation;
}
@@ -76,22 +134,27 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
entries.delete(server);
entries.set(server, entry);
while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!);
const pending = Promise.resolve().then(() => request(server)).then(result => {
if (!batch) snapshot?.refresh(server);
const latest = identity(server, probe, snapshot);
if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) {
throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.');
}
const observation = { result, observedAt: new Date().toISOString() };
if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) {
entry.identity = latest;
entry.observation = observation;
}
return observation;
}).finally(() => {
entry.pending = undefined;
if (entries.get(server) === entry && !entry.observation) entries.delete(server);
});
const pending = Promise.resolve()
.then(() => request(server))
.then((result) => {
if (!batch) snapshot?.refresh(server);
const latest = identity(server, probe, snapshot);
if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) {
throw new MonitoringChangedError(
'Server changed during monitoring; the previous observation was discarded.'
);
}
const observation = { result, observedAt: new Date().toISOString() };
if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) {
entry.identity = latest;
entry.observation = observation;
}
return observation;
})
.finally(() => {
entry.pending = undefined;
if (entries.get(server) === entry && !entry.observation) entries.delete(server);
});
entry.pending = pending;
return pending;
}
@@ -112,14 +175,19 @@ export function monitoringIdentity(server: string, snapshot?: MonitoringSnapshot
const container = identity(server, 'inspect', snapshot);
const game = identity(server, 'rcon', snapshot);
// Identity includes connection configuration; persist only its fingerprint.
return createHash('sha256').update(JSON.stringify([container.key, game.key, game.port])).digest('hex');
return createHash('sha256')
.update(JSON.stringify([container.key, game.key, game.port]))
.digest('hex');
}
/** Capture ownership before a status pass; publish only evidence for that instance. */
export function monitoringStatusWriter(server: string, snapshot?: MonitoringSnapshot) {
let before: string;
try { before = identity(server, 'inspect', snapshot).key; }
catch { return () => {}; }
try {
before = identity(server, 'inspect', snapshot).key;
} catch {
return () => {};
}
const slot = { snapshot: statuses.get(server)?.snapshot };
statuses.delete(server);
statuses.set(server, slot);
@@ -133,12 +201,17 @@ export function monitoringStatusWriter(server: string, snapshot?: MonitoringSnap
const current = slot.snapshot;
if (current && current.container.observedAt > container.observedAt) return;
slot.snapshot = { identity: monitoringIdentity(server, snapshot), container, game };
} catch { statuses.delete(server); }
} catch {
statuses.delete(server);
}
};
}
/** Latest completed status-monitor pass, with stale or superseded evidence excluded. */
export function monitoringStatus(server: string, snapshot?: MonitoringSnapshot): MonitoringStatus | null {
export function monitoringStatus(
server: string,
snapshot?: MonitoringSnapshot
): MonitoringStatus | null {
const saved = statuses.get(server)?.snapshot;
if (!saved) return null;
const fresh = (at: string) => {
@@ -146,43 +219,94 @@ export function monitoringStatus(server: string, snapshot?: MonitoringSnapshot):
return age >= 0 && age <= ONLINE_MAX_MS;
};
try {
if (saved.identity !== monitoringIdentity(server, snapshot)
|| !fresh(saved.container.observedAt)
|| newerEvidence(identity(server, 'inspect', snapshot), Date.parse(saved.container.observedAt))) return null;
const game = saved.game && fresh(saved.game.observedAt)
&& !newerEvidence(identity(server, 'rcon', snapshot), Date.parse(saved.game.observedAt)) ? saved.game : undefined;
if (
saved.identity !== monitoringIdentity(server, snapshot) ||
!fresh(saved.container.observedAt) ||
newerEvidence(
identity(server, 'inspect', snapshot),
Date.parse(saved.container.observedAt)
)
)
return null;
const game =
saved.game &&
fresh(saved.game.observedAt) &&
!newerEvidence(identity(server, 'rcon', snapshot), Date.parse(saved.game.observedAt))
? saved.game
: undefined;
return { ...saved, game };
} catch { return null; }
} catch {
return null;
}
}
/** Keep the actual sample time: a later heartbeat must not renew old evidence. */
export function heartbeatInspection(server: string, snapshot?: MonitoringSnapshot): Observation<Awaited<ReturnType<typeof nativeHealth>>> | null {
export function heartbeatInspection(
server: string,
snapshot?: MonitoringSnapshot
): Observation<Awaited<ReturnType<typeof nativeHealth>>> | null {
const row = snapshot ? snapshot.get(server)?.server : getServer(server);
if (!row?.native_instance_id || !row.host_id) return null;
const host = snapshot ? snapshot.get(server)?.host : getHost(row.host_id);
if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state')) return null;
const state = host.inventory.instanceStates?.find(state => state.instanceId === row.native_instance_id && state.generation === row.instance_generation);
if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state'))
return null;
const state = host.inventory.instanceStates?.find(
(state) =>
state.instanceId === row.native_instance_id &&
state.generation === row.instance_generation
);
if (!state) return null;
const at = Date.parse(state.observedAt);
const lifecycle = snapshot ? { at: snapshot.get(server)?.lifecycleAt } : db().prepare(`SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at
FROM server_deployments WHERE server=? AND server_deleted_at IS NULL`).get(server) as { at: string | null };
if (!reusable(at) || at < Date.parse(row.created_at) || at < Date.parse(lifecycle.at ?? '')
|| at < Date.parse((snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server)) ?? '') || at < Date.parse(row.container_checked_at ?? '')) return null;
return { observedAt: new Date(at).toISOString(), result: state.healthy
? { ok: true, state: state.state } : { ok: false, state: state.state, reason: `Container ${state.state}` } };
const lifecycle = snapshot
? { at: snapshot.get(server)?.lifecycleAt }
: (db()
.prepare(
`SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at
FROM server_deployments WHERE server=? AND server_deleted_at IS NULL`
)
.get(server) as { at: string | null });
if (
!reusable(at) ||
at < Date.parse(row.created_at) ||
at < Date.parse(lifecycle.at ?? '') ||
at <
Date.parse(
(snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server)) ?? ''
) ||
at < Date.parse(row.container_checked_at ?? '')
)
return null;
return {
observedAt: new Date(at).toISOString(),
result: state.healthy
? { ok: true, state: state.state }
: { ok: false, state: state.state, reason: `Container ${state.state}` }
};
}
type RconObservation = Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
/** Prime a whole host before the per-host queue admits just its first two servers. */
export function prepareMonitoringRcon(servers: string[], snapshot?: MonitoringSnapshot): Map<string, () => Promise<RconObservation>> {
export function prepareMonitoringRcon(
servers: string[],
snapshot?: MonitoringSnapshot
): Map<string, () => Promise<RconObservation>> {
const probes = new Map<string, () => Promise<RconObservation>>();
const batches = new Map<string, { instances: NativeInstance[]; result: Promise<{ results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> }> }>();
const batches = new Map<
string,
{
instances: NativeInstance[];
result: Promise<{
results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }>;
}>;
}
>();
for (const server of servers) {
const instance = snapshot ? snapshot.get(server)?.native : nativeInstance(server);
if (!instance) continue;
const host = snapshot ? snapshot.get(server)?.host : getHost(instance.host_id);
if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch')) continue;
if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch'))
continue;
const container = heartbeatInspection(server, snapshot);
if (container && !container.result.ok && container.result.state !== 'unknown') continue;
const before = identity(server, 'rcon', snapshot);
@@ -193,40 +317,73 @@ export function prepareMonitoringRcon(servers: string[], snapshot?: MonitoringSn
// observe() invokes each request in the same microtask turn;
// enqueue afterward so cached or shared probes need no batch slot.
const result = Promise.resolve().then(async () => {
const operation = withOperationSource('Status monitor', () => queueHostOperation(instance.host_id, 'rcon.status-batch', {
instances: instances.map(value => ({ instanceId: value.id, generation: value.generation }))
}));
const operation = withOperationSource('Status monitor', () =>
queueHostOperation(instance.host_id, 'rcon.status-batch', {
instances: instances.map((value) => ({
instanceId: value.id,
generation: value.generation
}))
})
);
// Allow delivery and receipt overhead beyond the agent's 20-second budget.
const receipt = await waitHostOperationStatus(operation.id, 25_000);
snapshot?.refresh();
if (receipt.state !== 'completed') throw new Error(receipt.error ?? 'Host status batch did not complete.');
return receipt.result as { results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> };
if (receipt.state !== 'completed')
throw new Error(receipt.error ?? 'Host status batch did not complete.');
return receipt.result as {
results: Array<{
instanceId: string;
generation: number;
reply: AgentRconReply;
}>;
};
});
batch = { instances, result }; batches.set(instance.host_id, batch);
batch = { instances, result };
batches.set(instance.host_id, batch);
}
batch.instances.push(instance);
const endpoint = { host: `agent:${instance.host_id}`, port: instance.port };
try {
const receipt = await batch.result;
const replies = receipt.results.filter(value => value.instanceId === instance.id && value.generation === instance.generation);
if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string') throw new Error('Host status batch omitted a valid instance result.');
if (replies[0].reply.reason === 'deadline') throw new MonitoringSkippedError('Host status batch budget expired before this server was checked.');
const replies = receipt.results.filter(
(value) =>
value.instanceId === instance.id && value.generation === instance.generation
);
if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string')
throw new Error('Host status batch omitted a valid instance result.');
if (replies[0].reply.reason === 'deadline')
throw new MonitoringSkippedError(
'Host status batch budget expired before this server was checked.'
);
return agentRconStatusResult(replies[0].reply, endpoint);
} catch (error) {
if (error instanceof MonitoringSkippedError) throw error;
return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message };
return {
ok: false as const,
endpoint,
reason: 'timeout' as const,
detail: (error as Error).message
};
}
};
// Observe before dispatch, including servers still waiting for their
// queue slot. Consume failures too, so a failed batch is never retried
// once per server or left as an unhandled rejected promise.
const pending = observe(server, false, 'rcon', rcon, request, snapshot, true).then(value => ({ value }), error => ({ error }));
const pending = observe(server, false, 'rcon', rcon, request, snapshot, true).then(
(value) => ({ value }),
(error) => ({ error })
);
probes.set(server, async () => {
const result = await pending;
if ('error' in result) throw result.error;
const current = identity(server, 'rcon', snapshot);
if (!sameIdentity(before, current) || newerEvidence(current, Date.parse(result.value.observedAt))) {
throw new MonitoringChangedError('Server changed before consuming the host status batch.');
if (
!sameIdentity(before, current) ||
newerEvidence(current, Date.parse(result.value.observedAt))
) {
throw new MonitoringChangedError(
'Server changed before consuming the host status batch.'
);
}
return result.value;
});
src/lib/server/fleet/health.ts
+7 -2
View File
@@ -9,9 +9,14 @@ const STALE_MAX_MS = 120_000;
* `stale` on a blocking RCON failure or if the last reply is >90-120s old.
* `online` if fresh, including the allowance for polling and delivery.
*/
export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
export function classifyHealth(
lastSeenAt: string | null,
lastError: string | null,
containerState?: string | null
): Health {
if (containerState === 'starting' || containerState === 'restarting') return 'starting';
if (containerState && containerState !== 'running' && containerState !== 'unknown') return 'offline';
if (containerState && containerState !== 'running' && containerState !== 'unknown')
return 'offline';
if (!lastSeenAt) return 'offline';
const age = Date.now() - new Date(lastSeenAt).getTime();
if (freshStatusTimestamp(lastSeenAt, Date.now())) {
src/lib/server/fleet/house-party.ts
+17 -5
View File
@@ -2,23 +2,35 @@ import { db } from '../db/index.js';
import { destroyServer } from '../provisioning/index.js';
export function housePartyTargets(): string[] {
return (db().prepare(`
return (
db()
.prepare(
`
SELECT name FROM servers WHERE destroyed_at IS NULL
UNION
SELECT server AS name FROM server_deployments
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
ORDER BY name
`).all() as { name: string }[]).map(row => row.name);
`
)
.all() as { name: string }[]
).map((row) => row.name);
}
export function queueHouseParty(expectedServers: string[]) {
return db().transaction(() => {
const servers = housePartyTargets();
const expected = [...expectedServers].sort();
if (servers.length !== expected.length || servers.some((name, index) => name !== expected[index])) {
return { ok: false as const, error: 'The fleet changed. Refresh the preview and review the servers again.' };
if (
servers.length !== expected.length ||
servers.some((name, index) => name !== expected[index])
) {
return {
ok: false as const,
error: 'The fleet changed. Refresh the preview and review the servers again.'
};
}
const deployments = servers.map(server => {
const deployments = servers.map((server) => {
const result = destroyServer(server, 'house-party-protocol');
if (!result.ok) throw new Error(result.message);
return { server, deploymentId: result.deploymentId! };
src/lib/server/fleet/inventory.ts
+3 -1
View File
@@ -36,7 +36,9 @@ export function recordScript(
}
export function removeScript(server: string, relPath: string): void {
db().prepare('DELETE FROM server_scripts WHERE server = ? AND rel_path = ?').run(server, relPath);
db()
.prepare('DELETE FROM server_scripts WHERE server = ? AND rel_path = ?')
.run(server, relPath);
}
export function getServerScript(server: string, relPath: string): ServerScriptRow | undefined {
src/lib/server/fleet/launch-recipes.ts
+248 -68
View File
@@ -1,12 +1,20 @@
import { packFileJson, unpackFileJson } from '../db/file-blobs.js';
import { randomUUID } from 'node:crypto';
import type { BlueprintSummary, LaunchRecipeSummary, LaunchFileSummary } from '$lib/deployment-plan.js';
import type {
BlueprintSummary,
LaunchRecipeSummary,
LaunchFileSummary
} from '$lib/deployment-plan.js';
import { config, configFileForGame } from '../config.js';
import { db } from '../db/index.js';
import { zombieMapBySlug, type ZombieGame } from './map-aliases.js';
import { promotedArtifactMetadata, promotedArtifacts } from './promoted-artifacts.js';
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
import { getBaselineForScope, getVersionMetadata, listPromotedOverlaysForScope } from './scripts.js';
import {
getBaselineForScope,
getVersionMetadata,
listPromotedOverlaysForScope
} from './scripts.js';
import { sha256Buffer } from './files.js';
export interface LaunchRecipePlan {
@@ -28,165 +36,337 @@ export interface LaunchRecipe extends LaunchRecipePlan {
files: (LaunchFileSummary & { content: string })[];
}
export interface BlueprintEntry { count: number; recipe: LaunchRecipePlan }
interface BlueprintSelection { count: number; game: ZombieGame; mapSlug: string }
interface BlueprintRow { id: string; name: string; description: string; created_at: string; recipe: string }
export interface BlueprintEntry {
count: number;
recipe: LaunchRecipePlan;
}
interface BlueprintSelection {
count: number;
game: ZombieGame;
mapSlug: string;
}
interface BlueprintRow {
id: string;
name: string;
description: string;
created_at: string;
recipe: string;
}
/** Legacy snapshots provide the lineup only; their file versions never govern a new launch. */
function readEntries(raw: string): BlueprintSelection[] {
const saved = JSON.parse(raw) as LaunchRecipe | { format: 2; entries: BlueprintEntry[] }
const saved = JSON.parse(raw) as
| LaunchRecipe
| { format: 2; entries: BlueprintEntry[] }
| { format: 3; entries: BlueprintSelection[] };
if (saved.format === 3) return saved.entries;
const entries = saved.format === 2 ? saved.entries : [{ count: 1, recipe: saved }];
return entries.map(({ count, recipe }) => ({ count, game: recipe.game, mapSlug: recipe.mapSlug }));
return entries.map(({ count, recipe }) => ({
count,
game: recipe.game,
mapSlug: recipe.mapSlug
}));
}
function summarizeBlueprint(row: BlueprintRow): BlueprintSummary {
const entries = readEntries(row.recipe).map(entry => ({ ...entry,
mapName: zombieMapBySlug(entry.game, entry.mapSlug)?.friendly ?? entry.mapSlug }));
return { id: row.id, name: row.name, description: row.description, createdAt: row.created_at,
entries, totalServers: entries.reduce((total, entry) => total + entry.count, 0) };
const entries = readEntries(row.recipe).map((entry) => ({
...entry,
mapName: zombieMapBySlug(entry.game, entry.mapSlug)?.friendly ?? entry.mapSlug
}));
return {
id: row.id,
name: row.name,
description: row.description,
createdAt: row.created_at,
entries,
totalServers: entries.reduce((total, entry) => total + entry.count, 0)
};
}
export function blueprintEntries(id: string): { entries: BlueprintEntry[]; blueprintName: string } {
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as BlueprintRow | undefined;
if (!row) throw new Error('This collection is no longer available. Choose another starting point.');
const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipePlan(game, mapSlug) }));
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as
BlueprintRow | undefined;
if (!row)
throw new Error('This collection is no longer available. Choose another starting point.');
const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({
count,
recipe: captureRecipePlan(game, mapSlug)
}));
return { entries, blueprintName: row.name };
}
function buildEntries(input: unknown): BlueprintSelection[] {
if (!Array.isArray(input) || input.length < 1 || input.length > 40) throw new Error('Add between 1 and 40 map selections.');
if (!Array.isArray(input) || input.length < 1 || input.length > 40)
throw new Error('Add between 1 and 40 map selections.');
const seen = new Set<string>();
const entries = input.map(entry => {
if (!entry || (entry.game !== 't5' && entry.game !== 't6') || typeof entry.mapSlug !== 'string'
|| !zombieMapBySlug(entry.game, entry.mapSlug)) throw new Error('Each selection needs a supported game and map.');
if (!Number.isSafeInteger(entry.count) || entry.count < 1 || entry.count > 15) throw new Error('Choose 1–15 lobbies for each map.');
const entries = input.map((entry) => {
if (
!entry ||
(entry.game !== 't5' && entry.game !== 't6') ||
typeof entry.mapSlug !== 'string' ||
!zombieMapBySlug(entry.game, entry.mapSlug)
)
throw new Error('Each selection needs a supported game and map.');
if (!Number.isSafeInteger(entry.count) || entry.count < 1 || entry.count > 15)
throw new Error('Choose 1–15 lobbies for each map.');
const key = `${entry.game}:${entry.mapSlug}`;
if (seen.has(key)) throw new Error('Each game/map can appear once. Increase its lobby count instead.');
if (seen.has(key))
throw new Error('Each game/map can appear once. Increase its lobby count instead.');
seen.add(key);
return { count: entry.count as number, game: entry.game as ZombieGame, mapSlug: entry.mapSlug as string };
return {
count: entry.count as number,
game: entry.game as ZombieGame,
mapSlug: entry.mapSlug as string
};
});
if (entries.reduce((total, entry) => total + entry.count, 0) > 100) throw new Error('A collection can launch at most 100 servers.');
if (entries.reduce((total, entry) => total + entry.count, 0) > 100)
throw new Error('A collection can launch at most 100 servers.');
return entries;
}
export function captureRecipe(game: ZombieGame, mapSlug: string, rconPassword?: string): LaunchRecipe {
export function captureRecipe(
game: ZombieGame,
mapSlug: string,
rconPassword?: string
): LaunchRecipe {
return buildRecipe(game, mapSlug, true, rconPassword);
}
/** Preview and admission keep file references; execution captures the current bytes. */
export function captureRecipePlan(game: ZombieGame, mapSlug: string, rconPassword?: string): LaunchRecipePlan {
export function captureRecipePlan(
game: ZombieGame,
mapSlug: string,
rconPassword?: string
): LaunchRecipePlan {
return buildRecipe(game, mapSlug, false, rconPassword);
}
function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: true, rconPassword?: string): LaunchRecipe;
function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: false, rconPassword?: string): LaunchRecipePlan;
function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean, rconPassword?: string): LaunchRecipePlan {
function buildRecipe(
game: ZombieGame,
mapSlug: string,
includeContent: true,
rconPassword?: string
): LaunchRecipe;
function buildRecipe(
game: ZombieGame,
mapSlug: string,
includeContent: false,
rconPassword?: string
): LaunchRecipePlan;
function buildRecipe(
game: ZombieGame,
mapSlug: string,
includeContent: boolean,
rconPassword?: string
): LaunchRecipePlan {
const map = zombieMapBySlug(game, mapSlug);
if (!map) throw new Error('Choose a supported game and map.');
const metadata = ({ kind, script, version }: ReturnType<typeof promotedArtifactMetadata>[number]): LaunchFileSummary => ({
name: script.name, kind, path: script.target_rel_path, versionId: version.id,
version: version.version, sha256: version.sha256, size: version.size
const metadata = ({
kind,
script,
version
}: ReturnType<typeof promotedArtifactMetadata>[number]): LaunchFileSummary => ({
name: script.name,
kind,
path: script.target_rel_path,
versionId: version.id,
version: version.version,
sha256: version.sha256,
size: version.size
});
const files: LaunchRecipePlan['files'] = includeContent
? promotedArtifacts(game, map.canonical).map(artifact => ({ ...metadata(artifact), content: artifact.version.content.toString('base64') }))
? promotedArtifacts(game, map.canonical).map((artifact) => ({
...metadata(artifact),
content: artifact.version.content.toString('base64')
}))
: promotedArtifactMetadata(game, map.canonical).map(metadata);
const addConfig = (name: string, path: string, content: Buffer) => files.push({
name, path, kind: 'config', versionId: null, version: null,
sha256: sha256Buffer(content), size: content.length, ...(includeContent ? { content: content.toString('base64') } : {})
});
const addConfig = (name: string, path: string, content: Buffer) =>
files.push({
name,
path,
kind: 'config',
versionId: null,
version: null,
sha256: sha256Buffer(content),
size: content.length,
...(includeContent ? { content: content.toString('base64') } : {})
});
const composed = getPromotedServerConfig(game, map.canonical, map.slug, rconPassword);
let configSource = 'Plutainer defaults, seeded on first boot';
if (composed) {
const baseline = getBaselineForScope('game', game)!;
const layers = [baseline, ...listPromotedOverlaysForScope('game', game), ...listPromotedOverlaysForScope('map', map.canonical)];
configSource = layers.map(layer => `${layer.name} v${getVersionMetadata(layer.current_version_id!)!.version}`).join(' + ');
const layers = [
baseline,
...listPromotedOverlaysForScope('game', game),
...listPromotedOverlaysForScope('map', map.canonical)
];
configSource = layers
.map(
(layer) =>
`${layer.name} v${getVersionMetadata(layer.current_version_id!)!.version}`
)
.join(' + ');
addConfig('Composed server configuration', configFileForGame(game), composed);
}
addConfig('Selected map', MAP_CONFIG_FILE, getServerMapConfig(game, map.canonical, map.slug)!);
return { format: 1, game, mapSlug, mapName: map.friendly, expectedMap: map.canonical,
capturedAt: new Date().toISOString(), image: config.defaultRuntimeImage,
configFile: configFileForGame(game), configSource, files };
return {
format: 1,
game,
mapSlug,
mapName: map.friendly,
expectedMap: map.canonical,
capturedAt: new Date().toISOString(),
image: config.defaultRuntimeImage,
configFile: configFileForGame(game),
configSource,
files
};
}
export function summarizeRecipe(recipe: LaunchRecipePlan, fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe)))): LaunchRecipeSummary {
export function summarizeRecipe(
recipe: LaunchRecipePlan,
fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe)))
): LaunchRecipeSummary {
const { format: _format, files, ...summary } = recipe;
return { ...summary, files: files.map(({ content: _content, ...file }) => file),
fingerprint };
return { ...summary, files: files.map(({ content: _content, ...file }) => file), fingerprint };
}
/** Keep the small detail response beside the immutable attempted file bytes. */
export function recipeStorage(recipe: LaunchRecipePlan): { recipe: string; recipeSummary: string } {
const encoded = JSON.stringify(recipe);
return { recipe: JSON.stringify(packFileJson(recipe).value), recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) };
return {
recipe: JSON.stringify(packFileJson(recipe).value),
recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded))))
};
}
export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
const row = db().prepare('SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
export function recipeSummaryForDeployment(
id: string
): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
const row = db()
.prepare(
'SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?'
)
.get(id) as { recipe_summary: string | null; blueprint_name: string | null } | undefined;
if (!row) return null;
if (row.recipe_summary !== null) return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name };
if (row.recipe_summary !== null)
return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name };
// Old snapshots remain readable. Populate on first access, without making
// controller startup scan and decode every retained plugin in its history.
const launch = recipeForDeployment(id)!;
const summary = summarizeRecipe(launch.recipe);
db().prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?').run(JSON.stringify(summary), id);
db()
.prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?')
.run(JSON.stringify(summary), id);
return { recipe: summary, blueprintName: row.blueprint_name };
}
export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
const row = db().prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
export function recipeForDeployment(
id: string
): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
const row = db()
.prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
.get(id) as { recipe: string; blueprint_name: string | null } | undefined;
return row ? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name } : null;
return row
? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name }
: null;
}
export function blueprintRecipe(id: string): { recipe: LaunchRecipePlan; blueprintName: string } {
const { entries, blueprintName } = blueprintEntries(id);
if (entries.length !== 1 || entries[0].count !== 1) throw new Error('This collection launches multiple servers. Use the collection preview.');
if (entries.length !== 1 || entries[0].count !== 1)
throw new Error('This collection launches multiple servers. Use the collection preview.');
return { recipe: entries[0].recipe, blueprintName };
}
export function listBlueprints(): BlueprintSummary[] {
return (db().prepare('SELECT * FROM server_blueprints ORDER BY name COLLATE NOCASE').all() as BlueprintRow[]).map(summarizeBlueprint);
return (
db()
.prepare('SELECT * FROM server_blueprints ORDER BY name COLLATE NOCASE')
.all() as BlueprintRow[]
).map(summarizeBlueprint);
}
function blueprintName(input: { name: unknown; description?: unknown }, exceptId = ''): string {
if (typeof input.name !== 'string' || !input.name.trim() || input.name.trim().length > 80) throw new Error('Give the collection a name of 1–80 characters.');
if (input.description !== undefined && (typeof input.description !== 'string' || input.description.length > 500)) throw new Error('Keep the description to 500 characters.');
if (db().prepare('SELECT id FROM server_blueprints WHERE name = ? COLLATE NOCASE AND id != ?').get(input.name.trim(), exceptId)) throw new Error('A collection with this name already exists. Choose a new name.');
if (typeof input.name !== 'string' || !input.name.trim() || input.name.trim().length > 80)
throw new Error('Give the collection a name of 1–80 characters.');
if (
input.description !== undefined &&
(typeof input.description !== 'string' || input.description.length > 500)
)
throw new Error('Keep the description to 500 characters.');
if (
db()
.prepare('SELECT id FROM server_blueprints WHERE name = ? COLLATE NOCASE AND id != ?')
.get(input.name.trim(), exceptId)
)
throw new Error('A collection with this name already exists. Choose a new name.');
return input.name.trim();
}
export function saveBlueprint(input: { name: unknown; description?: unknown; previewId?: unknown; deploymentId?: unknown; entries?: unknown }): BlueprintSummary {
export function saveBlueprint(input: {
name: unknown;
description?: unknown;
previewId?: unknown;
deploymentId?: unknown;
entries?: unknown;
}): BlueprintSummary {
const name = blueprintName(input);
if ([input.previewId, input.deploymentId, input.entries].filter(value => value !== undefined).length !== 1) throw new Error('Choose exactly one launch preview, recorded deployment, or set of maps.');
if (
[input.previewId, input.deploymentId, input.entries].filter((value) => value !== undefined)
.length !== 1
)
throw new Error('Choose exactly one launch preview, recorded deployment, or set of maps.');
let recipe: LaunchRecipePlan | undefined;
if (typeof input.previewId === 'string') {
const row = db().prepare('SELECT recipe FROM server_launch_previews WHERE id = ? AND expires_at > ?')
const row = db()
.prepare('SELECT recipe FROM server_launch_previews WHERE id = ? AND expires_at > ?')
.get(input.previewId, new Date().toISOString()) as { recipe: string } | undefined;
if (row) recipe = JSON.parse(row.recipe);
} else if (typeof input.deploymentId === 'string') recipe = recipeForDeployment(input.deploymentId)?.recipe;
if (!recipe && input.entries === undefined) throw new Error('The launch recipe is unavailable or the preview expired. Generate a fresh preview.');
const entries = input.entries !== undefined ? buildEntries(input.entries)
: buildEntries([{ count: 1, game: recipe!.game, mapSlug: recipe!.mapSlug }]);
} else if (typeof input.deploymentId === 'string')
recipe = recipeForDeployment(input.deploymentId)?.recipe;
if (!recipe && input.entries === undefined)
throw new Error(
'The launch recipe is unavailable or the preview expired. Generate a fresh preview.'
);
const entries =
input.entries !== undefined
? buildEntries(input.entries)
: buildEntries([{ count: 1, game: recipe!.game, mapSlug: recipe!.mapSlug }]);
const id = randomUUID();
const createdAt = new Date().toISOString();
const saved = JSON.stringify({ format: 3, entries });
db().prepare('INSERT INTO server_blueprints (id, name, description, created_at, recipe) VALUES (?, ?, ?, ?, ?)')
db()
.prepare(
'INSERT INTO server_blueprints (id, name, description, created_at, recipe) VALUES (?, ?, ?, ?, ?)'
)
.run(id, name, input.description ?? '', createdAt, saved);
return summarizeBlueprint({ id, name, description: (input.description as string) ?? '', created_at: createdAt, recipe: saved });
return summarizeBlueprint({
id,
name,
description: (input.description as string) ?? '',
created_at: createdAt,
recipe: saved
});
}
export function updateBlueprint(id: string, input: { name: unknown; description?: unknown; entries?: unknown }): BlueprintSummary {
export function updateBlueprint(
id: string,
input: { name: unknown; description?: unknown; entries?: unknown }
): BlueprintSummary {
return db().transaction(() => {
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as BlueprintRow | undefined;
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as
BlueprintRow | undefined;
if (!row) throw new Error('Collection not found.');
const name = blueprintName(input, id);
const entries = buildEntries(input.entries);
const recipe = JSON.stringify({ format: 3, entries });
const description = (input.description as string) ?? '';
db().prepare('UPDATE server_blueprints SET name = ?, description = ?, recipe = ? WHERE id = ?').run(name, description, recipe, id);
db()
.prepare(
'UPDATE server_blueprints SET name = ?, description = ?, recipe = ? WHERE id = ?'
)
.run(name, description, recipe, id);
return summarizeBlueprint({ ...row, name, description, recipe });
})();
}
src/lib/server/fleet/leaderboard-ingest.ts
+133 -37
View File
@@ -20,7 +20,9 @@ interface Participation {
}
function mapIdentity(handle: Database.Database, report: GameTelemetry) {
const server = handle.prepare('SELECT game, map, map_slug FROM servers WHERE name = ?').get(report.server) as
const server = handle
.prepare('SELECT game, map, map_slug FROM servers WHERE name = ?')
.get(report.server) as
{ game: string | null; map: string | null; map_slug: string | null } | undefined;
const parsed = parseServerName(report.server);
const game = (server?.game ?? parsed?.game ?? 'unknown').toLowerCase();
@@ -28,15 +30,22 @@ function mapIdentity(handle: Database.Database, report: GameTelemetry) {
// A shared engine map (notably zm_transit) is not enough to identify T6's
// location/ruleset. Use the registered selection only when it matches.
const parsedSlug = parsed?.mapCanonical === map ? parsed.mapSlug : undefined;
const slug = server?.map === map ? server.map_slug ?? parsedSlug : parsedSlug;
const candidates = zombieMaps(game).filter(entry => entry.canonical === map);
const selected = game === 't6' && candidates.length > 1 && !slug
? undefined : selectedZombieMap(game, map, slug);
const variant = game === 't6' ? selected?.slug ?? slug ?? 'unspecified' : '';
const slug = server?.map === map ? (server.map_slug ?? parsedSlug) : parsedSlug;
const candidates = zombieMaps(game).filter((entry) => entry.canonical === map);
const selected =
game === 't6' && candidates.length > 1 && !slug
? undefined
: selectedZombieMap(game, map, slug);
const variant = game === 't6' ? (selected?.slug ?? slug ?? 'unspecified') : '';
return {
game, map,
game,
map,
key: [game, map, ...(variant ? [variant] : [])].join(':'),
name: selected?.friendly ?? (game === 't6' && candidates.length > 1 ? `${map} (${slug ?? 'unspecified variant'})` : map)
name:
selected?.friendly ??
(game === 't6' && candidates.length > 1
? `${map} (${slug ?? 'unspecified variant'})`
: map)
};
}
@@ -44,48 +53,92 @@ function fingerprint(report: GameTelemetry): string {
// Receipt timestamps and roster ordering do not identify new gameplay.
const { receivedAt: _receivedAt, timing: _timing, ...event } = report;
const players = [...event.players].sort((a, b) => a.guid.localeCompare(b.guid));
return createHash('sha256').update(JSON.stringify({ ...event, players })).digest('hex');
return createHash('sha256')
.update(JSON.stringify({ ...event, players }))
.digest('hex');
}
/** Called inside the same transaction as the latest scoreboard snapshot. */
export function recordLeaderboardTelemetry(handle: Database.Database, report: GameTelemetry): void {
const identity = mapIdentity(handle, report);
const signature = fingerprint(report);
let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1')
let match = handle
.prepare(
'SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1'
)
.get(report.server) as Match | undefined;
if (match?.last_fingerprint === signature && match.map_key === identity.key) return;
const isStart = report.event === 'game_start';
const round = isStart ? 0 : report.round;
const newMatch = !match || match.map_key !== identity.key
|| (isStart && (match.status !== 'active' || match.last_event !== 'game_start'))
|| (!isStart && round < match.highest_round)
|| ((report.event === 'round_change' || report.event === 'heartbeat') && match.status !== 'active');
const newMatch =
!match ||
match.map_key !== identity.key ||
(isStart && (match.status !== 'active' || match.last_event !== 'game_start')) ||
(!isStart && round < match.highest_round) ||
((report.event === 'round_change' || report.event === 'heartbeat') &&
match.status !== 'active');
if (newMatch) {
if (match?.status === 'active') {
handle.prepare("UPDATE leaderboard_matches SET status = 'interrupted', ended_at = ? WHERE id = ?")
handle
.prepare(
"UPDATE leaderboard_matches SET status = 'interrupted', ended_at = ? WHERE id = ?"
)
.run(report.receivedAt, match.id);
}
const inserted = handle.prepare(`INSERT INTO leaderboard_matches
const inserted = handle
.prepare(
`INSERT INTO leaderboard_matches
(server, game, map, map_key, map_name, started_at, last_received_at, start_observed, last_event, last_fingerprint)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
.run(report.server, identity.game, identity.map, identity.key, identity.name,
report.receivedAt, report.receivedAt, Number(isStart), report.event, signature);
match = { id: Number(inserted.lastInsertRowid), game: identity.game, map_key: identity.key,
status: 'active', highest_round: 0, last_event: report.event, last_fingerprint: signature };
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(
report.server,
identity.game,
identity.map,
identity.key,
identity.name,
report.receivedAt,
report.receivedAt,
Number(isStart),
report.event,
signature
);
match = {
id: Number(inserted.lastInsertRowid),
game: identity.game,
map_key: identity.key,
status: 'active',
highest_round: 0,
last_event: report.event,
last_fingerprint: signature
};
}
if (!match) return;
if (isStart) {
handle.prepare('UPDATE leaderboard_matches SET start_players = COALESCE(start_players, ?) WHERE id = ?')
handle
.prepare(
'UPDATE leaderboard_matches SET start_players = COALESCE(start_players, ?) WHERE id = ?'
)
.run(report.players.length, match.id);
}
const previous = new Map((handle.prepare('SELECT guid, last_round, present FROM leaderboard_match_players WHERE match_id = ?')
.all(match.id) as Participation[]).map(player => [player.guid, player]));
handle.prepare('UPDATE leaderboard_match_players SET present = 0 WHERE match_id = ?').run(match.id);
const saveIdentity = handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
const previous = new Map(
(
handle
.prepare(
'SELECT guid, last_round, present FROM leaderboard_match_players WHERE match_id = ?'
)
.all(match.id) as Participation[]
).map((player) => [player.guid, player])
);
handle
.prepare('UPDATE leaderboard_match_players SET present = 0 WHERE match_id = ?')
.run(match.id);
const saveIdentity =
handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET
name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END,
last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`);
@@ -108,33 +161,76 @@ export function recordLeaderboardTelemetry(handle: Database.Database, report: Ga
const completedRounds = Math.max(0, round - 1);
for (const player of report.players) {
const prior = previous.get(player.guid);
const survived = !isStart && prior?.present ? Math.max(0, completedRounds - prior.last_round) : 0;
saveIdentity.run(identity.game, player.guid, player.name, report.receivedAt, report.receivedAt);
const survived =
!isStart && prior?.present ? Math.max(0, completedRounds - prior.last_round) : 0;
saveIdentity.run(
identity.game,
player.guid,
player.name,
report.receivedAt,
report.receivedAt
);
const stats = 'kills' in player ? player : null;
savePlayer.run(match.id, player.guid, round, survived,
stats?.kills ?? 0, stats?.headshots ?? 0, stats?.downs ?? 0, stats?.revives ?? 0, stats?.score ?? 0,
completedRounds, Number(!isStart), Number(report.event === 'game_end'), report.receivedAt, report.receivedAt, round, Number(isStart));
savePlayer.run(
match.id,
player.guid,
round,
survived,
stats?.kills ?? 0,
stats?.headshots ?? 0,
stats?.downs ?? 0,
stats?.revives ?? 0,
stats?.score ?? 0,
completedRounds,
Number(!isStart),
Number(report.event === 'game_end'),
report.receivedAt,
report.receivedAt,
round,
Number(isStart)
);
}
handle.prepare(`UPDATE leaderboard_matches SET highest_round = MAX(highest_round, ?),
handle
.prepare(
`UPDATE leaderboard_matches SET highest_round = MAX(highest_round, ?),
last_received_at = ?, last_event = ?, last_fingerprint = ?,
status = CASE WHEN ? = 'game_end' THEN 'completed' ELSE status END,
ended_at = CASE WHEN ? = 'game_end' THEN ? ELSE ended_at END WHERE id = ?`)
.run(round, report.receivedAt, report.event, signature, report.event, report.event, report.receivedAt, match.id);
ended_at = CASE WHEN ? = 'game_end' THEN ? ELSE ended_at END WHERE id = ?`
)
.run(
round,
report.receivedAt,
report.event,
signature,
report.event,
report.event,
report.receivedAt,
match.id
);
}
/** Seed the single surviving scoreboard per server once; older games are gone. */
export function backfillLeaderboard(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'leaderboard_v1'").get()) return;
handle.transaction(() => {
const reports = handle.prepare(`SELECT payload FROM server_telemetry WHERE payload IS NOT NULL ORDER BY received_at, server`)
const reports = handle
.prepare(
`SELECT payload FROM server_telemetry WHERE payload IS NOT NULL ORDER BY received_at, server`
)
.all() as Array<{ payload: string }>;
for (const row of reports) {
let report: GameTelemetry;
try {
const stored = JSON.parse(row.payload);
if (typeof stored.receivedAt !== 'string' || !Number.isFinite(Date.parse(stored.receivedAt))) continue;
if (
typeof stored.receivedAt !== 'string' ||
!Number.isFinite(Date.parse(stored.receivedAt))
)
continue;
report = { ...parseGameEvent(stored), receivedAt: stored.receivedAt };
} catch { continue; } // Legacy malformed snapshots cannot establish player records.
} catch {
continue;
} // Legacy malformed snapshots cannot establish player records.
recordLeaderboardTelemetry(handle, report);
}
handle.prepare("INSERT INTO settings (key, value) VALUES ('leaderboard_v1', '1')").run();
src/lib/server/fleet/leaderboard-records.ts
+39 -12
View File
@@ -1,5 +1,10 @@
import type Database from 'better-sqlite3';
import type { LeaderboardPlayerCount, PersonalBestMetric, PersonalBestRecords, RecordEvidence } from '../../leaderboard.js';
import type {
LeaderboardPlayerCount,
PersonalBestMetric,
PersonalBestRecords,
RecordEvidence
} from '../../leaderboard.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
interface RecordRequest extends Record<PersonalBestMetric, number> {
@@ -10,14 +15,27 @@ interface RecordRequest extends Record<PersonalBestMetric, number> {
}
/** Fetch evidence only for requested players and scopes, never rank the whole fleet's history. */
export function getPersonalBestRecords(handle: Database.Database, requests: RecordRequest[]): PersonalBestRecords[] {
export function getPersonalBestRecords(
handle: Database.Database,
requests: RecordRequest[]
): PersonalBestRecords[] {
const result: PersonalBestRecords[] = [];
// Page requests are at most 100 players; chunk unusually large profile map collections too.
for (let offset = 0; offset < requests.length; offset += 100) {
const batch = requests.slice(offset, offset + 100);
const parameters = batch.flatMap((request, index) => [index, request.guid, request.game ?? null,
request.map ?? null, request.players ?? null, request.highestRound, request.bestKills, request.bestScore]);
const rows = handle.prepare(`WITH requested (requestId, guid, game, map, players, highestRound, bestKills, bestScore) AS (
const parameters = batch.flatMap((request, index) => [
index,
request.guid,
request.game ?? null,
request.map ?? null,
request.players ?? null,
request.highestRound,
request.bestKills,
request.bestScore
]);
const rows = handle
.prepare(
`WITH requested (requestId, guid, game, map, players, highestRound, bestKills, bestScore) AS (
VALUES ${batch.map(() => '(?, ?, ?, ?, ?, ?, ?, ?)').join(', ')}
), candidates AS (
SELECT r.requestId, m.id AS matchId, m.game, m.map_key AS mapKey, m.map_name AS mapName,
@@ -41,14 +59,23 @@ export function getPersonalBestRecords(handle: Database.Database, requests: Reco
SELECT *, ROW_NUMBER() OVER (PARTITION BY requestId, metric ORDER BY completionOrder, matchId) AS choice
FROM valuesByMetric
) SELECT requestId, metric, value, matchId, game, mapKey, mapName, server, startedAt, endedAt,
startObserved, firstObservedRound, presentAtStart FROM ranked WHERE choice = 1`)
.all(...parameters) as Array<Omit<RecordEvidence, 'startObserved' | 'presentAtStart'> & {
requestId: number; metric: PersonalBestMetric; startObserved: number; presentAtStart: number | null;
}>;
const records = batch.map(() => ({} as PersonalBestRecords));
startObserved, firstObservedRound, presentAtStart FROM ranked WHERE choice = 1`
)
.all(...parameters) as Array<
Omit<RecordEvidence, 'startObserved' | 'presentAtStart'> & {
requestId: number;
metric: PersonalBestMetric;
startObserved: number;
presentAtStart: number | null;
}
>;
const records = batch.map(() => ({}) as PersonalBestRecords);
for (const { requestId, metric, startObserved, presentAtStart, ...record } of rows) {
records[requestId][metric] = { ...record, startObserved: Boolean(startObserved),
presentAtStart: presentAtStart === null ? null : Boolean(presentAtStart) };
records[requestId][metric] = {
...record,
startObserved: Boolean(startObserved),
presentAtStart: presentAtStart === null ? null : Boolean(presentAtStart)
};
}
result.push(...records);
}
src/lib/server/fleet/leaderboard.ts
+166 -54
View File
@@ -2,19 +2,34 @@ import { db } from '../db/index.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
import { getPersonalBestRecords } from './leaderboard-records.js';
import { countsAsGamePlayed } from './leaderboard-participation.js';
import { EFFICIENCY_MIN_GAMES, EFFICIENCY_MIN_KILLS, EFFICIENCY_MIN_DOWNS,
isEfficiencyMetric } from '../../leaderboard-efficiency.js';
import { leaderboardMetrics, type LeaderboardEntry, type LeaderboardMap, type LeaderboardMetric,
type LeaderboardOptions, type LeaderboardResponse } from '../../leaderboard.js';
import {
EFFICIENCY_MIN_GAMES,
EFFICIENCY_MIN_KILLS,
EFFICIENCY_MIN_DOWNS,
isEfficiencyMetric
} from '../../leaderboard-efficiency.js';
import {
leaderboardMetrics,
type LeaderboardEntry,
type LeaderboardMap,
type LeaderboardMetric,
type LeaderboardOptions,
type LeaderboardResponse
} from '../../leaderboard.js';
const metrics = new Set<string>(leaderboardMetrics.map(metric => metric.key));
const metrics = new Set<string>(leaderboardMetrics.map((metric) => metric.key));
export const MAX_LEADERBOARD_OFFSET = 1_000_000;
type RankedEntry = Omit<LeaderboardEntry, 'records'>;
interface RankedScope { revision: number; entries: RankedEntry[]; maps: LeaderboardMap[]; summary: LeaderboardResponse['summary'] }
interface RankedScope {
revision: number;
entries: RankedEntry[];
maps: LeaderboardMap[];
summary: LeaderboardResponse['summary'];
}
const scopeCaches = new WeakMap<object, Map<string, RankedScope>>();
// Match SQLite's built-in lower(), which folds ASCII rather than Unicode.
const searchText = (value: string) => value.replace(/[A-Z]/g, ch => ch.toLowerCase());
const searchText = (value: string) => value.replace(/[A-Z]/g, (ch) => ch.toLowerCase());
interface LeaderboardRow extends Omit<RankedEntry, 'guid'> {
// A left join keeps metadata available when the requested page has no players.
@@ -39,35 +54,78 @@ export function getLeaderboard(options: LeaderboardOptions = {}): LeaderboardRes
}
/** Select the exact player after ranking everyone eligible in the requested scope. */
export function getPlayerLeaderboardEntry(guid: string,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players' | 'metric'> = {}): LeaderboardEntry | null {
return readLeaderboard({ game: options.game, map: options.map, players: options.players, metric: options.metric, limit: 1 }, guid);
export function getPlayerLeaderboardEntry(
guid: string,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players' | 'metric'> = {}
): LeaderboardEntry | null {
return readLeaderboard(
{
game: options.game,
map: options.map,
players: options.players,
metric: options.metric,
limit: 1
},
guid
);
}
function addRecordSources(handle: ReturnType<typeof db>, entries: RankedEntry[], options: LeaderboardOptions): LeaderboardEntry[] {
const records = getPersonalBestRecords(handle, entries.map(entry => ({
...entry, game: options.game || undefined, map: options.map || undefined, players: options.players
})));
function addRecordSources(
handle: ReturnType<typeof db>,
entries: RankedEntry[],
options: LeaderboardOptions
): LeaderboardEntry[] {
const records = getPersonalBestRecords(
handle,
entries.map((entry) => ({
...entry,
game: options.game || undefined,
map: options.map || undefined,
players: options.players
}))
);
return entries.map((entry, index) => ({ ...entry, records: records[index] }));
}
function readLeaderboard(options: LeaderboardOptions): LeaderboardResponse;
function readLeaderboard(options: LeaderboardOptions, exactGuid: string): LeaderboardEntry | null;
function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): LeaderboardResponse | LeaderboardEntry | null {
function readLeaderboard(
options: LeaderboardOptions,
exactGuid?: string
): LeaderboardResponse | LeaderboardEntry | null {
const handle = db();
const metric = options.metric && isLeaderboardMetric(options.metric) ? options.metric : 'highestRound';
const limit = Number.isSafeInteger(options.limit) ? Math.min(100, Math.max(1, options.limit!)) : 50;
const offset = Number.isSafeInteger(options.offset) ? Math.min(MAX_LEADERBOARD_OFFSET, Math.max(0, options.offset!)) : 0;
const conditions = ["m.game IN ('t5', 't6')", "m.status = 'completed'", 'mp.has_stats = 1', `m.highest_round >= ${MIN_RECORDED_ROUND}`,
`mp.highest_round >= ${MIN_RECORDED_ROUND}`];
const metric =
options.metric && isLeaderboardMetric(options.metric) ? options.metric : 'highestRound';
const limit = Number.isSafeInteger(options.limit)
? Math.min(100, Math.max(1, options.limit!))
: 50;
const offset = Number.isSafeInteger(options.offset)
? Math.min(MAX_LEADERBOARD_OFFSET, Math.max(0, options.offset!))
: 0;
const conditions = [
"m.game IN ('t5', 't6')",
"m.status = 'completed'",
'mp.has_stats = 1',
`m.highest_round >= ${MIN_RECORDED_ROUND}`,
`mp.highest_round >= ${MIN_RECORDED_ROUND}`
];
const parameters: Array<string | number> = [];
if (options.game) { conditions.push('m.game = ?'); parameters.push(options.game); }
if (options.map) { conditions.push('m.map_key = ?'); parameters.push(options.map); }
if (options.players) { conditions.push('m.start_players = ?'); parameters.push(options.players); }
if (options.game) {
conditions.push('m.game = ?');
parameters.push(options.game);
}
if (options.map) {
conditions.push('m.map_key = ?');
parameters.push(options.map);
}
if (options.players) {
conditions.push('m.start_players = ?');
parameters.push(options.players);
}
const qualification = isEfficiencyMetric(metric)
? `WHERE measures.gamesPlayed >= ${EFFICIENCY_MIN_GAMES} AND measures.${metric} IS NOT NULL`
+ (metric === 'headshotRate' ? ` AND measures.kills >= ${EFFICIENCY_MIN_KILLS}` : '')
+ (metric === 'killsPerDown' ? ` AND measures.downs >= ${EFFICIENCY_MIN_DOWNS}` : '')
? `WHERE measures.gamesPlayed >= ${EFFICIENCY_MIN_GAMES} AND measures.${metric} IS NOT NULL` +
(metric === 'headshotRate' ? ` AND measures.kills >= ${EFFICIENCY_MIN_KILLS}` : '') +
(metric === 'killsPerDown' ? ` AND measures.downs >= ${EFFICIENCY_MIN_DOWNS}` : '')
: '';
// A board reuses observations for player totals and distinct match metadata.
// Exact-player requests only consume the totals, so they can stream the join once.
@@ -100,14 +158,31 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
FROM measures JOIN identities p ON p.guid = measures.guid AND p.recency = 1 ${qualification}
)`;
let revision = -1;
try { revision = (handle.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'").get() as { revision: number }).revision; }
catch { /* Standalone history tools can operate without cache metadata. */ }
try {
revision = (
handle
.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'")
.get() as { revision: number }
).revision;
} catch {
/* Standalone history tools can operate without cache metadata. */
}
let cache = scopeCaches.get(handle);
if (!cache) { cache = new Map(); scopeCaches.set(handle, cache); }
const cacheKey = JSON.stringify([options.game ?? '', options.map ?? '', options.players ?? '', metric]);
if (!cache) {
cache = new Map();
scopeCaches.set(handle, cache);
}
const cacheKey = JSON.stringify([
options.game ?? '',
options.map ?? '',
options.players ?? '',
metric
]);
let saved = cache.get(cacheKey);
if (!saved || revision < 0 || saved.revision !== revision) {
const rows = handle.prepare(`${aggregated}, playerSummary AS (
const rows = handle
.prepare(
`${aggregated}, playerSummary AS (
SELECT COUNT(*) AS players, COALESCE(SUM(kills), 0) AS kills,
COALESCE(SUM(roundsSurvived), 0) AS roundsSurvived,
COALESCE(MAX(highestRound), 0) AS highestRound FROM totals
@@ -121,36 +196,73 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
metadata.roundsSurvived AS summaryRounds, metadata.highestRound AS summaryHighestRound,
metadata.firstTrackedAt AS summaryFirstTrackedAt, metadata.updatedAt AS summaryUpdatedAt
FROM metadata LEFT JOIN ranked ON 1
ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`)
.all(...parameters) as LeaderboardRow[];
const rankedEntries = rows.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
.map(({ matchingTotal, summaryPlayers, summaryMatches, summaryKills, summaryRounds,
summaryHighestRound, summaryFirstTrackedAt, summaryUpdatedAt, ...entry }) => entry);
const metadata = rows[0];
const summary: LeaderboardResponse['summary'] = {
players: metadata.summaryPlayers, matches: metadata.summaryMatches, kills: metadata.summaryKills,
roundsSurvived: metadata.summaryRounds, highestRound: metadata.summaryHighestRound,
firstTrackedAt: metadata.summaryFirstTrackedAt, updatedAt: metadata.summaryUpdatedAt
};
// Keep qualifying map variants available, even after their server is removed.
const maps = handle.prepare(`SELECT map_key AS key, game, map, map_name AS name
ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`
)
.all(...parameters) as LeaderboardRow[];
const rankedEntries = rows
.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
.map(
({
matchingTotal,
summaryPlayers,
summaryMatches,
summaryKills,
summaryRounds,
summaryHighestRound,
summaryFirstTrackedAt,
summaryUpdatedAt,
...entry
}) => entry
);
const metadata = rows[0];
const summary: LeaderboardResponse['summary'] = {
players: metadata.summaryPlayers,
matches: metadata.summaryMatches,
kills: metadata.summaryKills,
roundsSurvived: metadata.summaryRounds,
highestRound: metadata.summaryHighestRound,
firstTrackedAt: metadata.summaryFirstTrackedAt,
updatedAt: metadata.summaryUpdatedAt
};
// Keep qualifying map variants available, even after their server is removed.
const maps = handle
.prepare(
`SELECT map_key AS key, game, map, map_name AS name
FROM leaderboard_matches WHERE game IN ('t5', 't6') AND status = 'completed'
AND highest_round >= ${MIN_RECORDED_ROUND}${options.game ? ' AND game = ?' : ''}
GROUP BY game, map_key HAVING id = MAX(id)
ORDER BY game, name COLLATE NOCASE, map_key`)
.all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
saved = { revision, entries: rankedEntries, maps, summary };
cache.delete(cacheKey);
if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
// Bound total retained players as well as the number of scopes.
while (cache.size > 8 || [...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000) cache.delete(cache.keys().next().value!);
ORDER BY game, name COLLATE NOCASE, map_key`
)
.all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
saved = { revision, entries: rankedEntries, maps, summary };
cache.delete(cacheKey);
if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
// Bound total retained players as well as the number of scopes.
while (
cache.size > 8 ||
[...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000
)
cache.delete(cache.keys().next().value!);
}
if (exactGuid !== undefined) {
const entry = saved.entries.find(entry => entry.guid === exactGuid);
const entry = saved.entries.find((entry) => entry.guid === exactGuid);
return entry ? addRecordSources(handle, [entry], options)[0] : null;
}
const query = searchText(options.q?.trim() ?? '');
const matching = query ? saved.entries.filter(entry => searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)) : saved.entries;
const matching = query
? saved.entries.filter(
(entry) =>
searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)
)
: saved.entries;
const entries = addRecordSources(handle, matching.slice(offset, offset + limit), options);
return { entries, maps: saved.maps, total: matching.length, limit, offset, metric, summary: saved.summary };
return {
entries,
maps: saved.maps,
total: matching.length,
limit,
offset,
metric,
summary: saved.summary
};
}
src/lib/server/fleet/library-metadata.ts
+64 -17
View File
@@ -6,7 +6,9 @@ export function scriptCatalog(kind?: ArtifactKind) {
const version = (alias: string) => `CASE WHEN ${alias}.id IS NOT NULL THEN json_object(
'id',${alias}.id,'version',${alias}.version,'release_version',${alias}.release_version,
'sha256',${alias}.sha256,'size',${alias}.size,'state',${alias}.state,'uploaded_at',${alias}.uploaded_at) END`;
const rows = db().prepare(`SELECT s.*, ${version('c')} current_json, ${version('v')} latest_json,
const rows = db()
.prepare(
`SELECT s.*, ${version('c')} current_json, ${version('v')} latest_json,
COALESCE(counts.n,0) versionCount, active.beta_id runningBetaId
FROM scripts s
LEFT JOIN (SELECT script_id, COUNT(*) n, MAX(version) latest FROM script_versions GROUP BY script_id) counts ON counts.script_id=s.id
@@ -14,32 +16,77 @@ export function scriptCatalog(kind?: ArtifactKind) {
LEFT JOIN script_versions c ON c.id=s.current_version_id
LEFT JOIN (SELECT bs.script_id, MIN(b.id) beta_id FROM beta_scripts bs JOIN betas b ON b.id=bs.beta_id
WHERE b.state IN ('running','ready-promote') GROUP BY bs.script_id) active ON active.script_id=s.id
${kind ? 'WHERE s.kind=?' : ''} ORDER BY s.name`).all(...(kind ? [kind] : [])) as Array<ScriptRow & {
current_json: string | null; latest_json: string | null; versionCount: number; runningBetaId: number | null;
}>;
const servers = db().prepare('SELECT name, game, map FROM servers ORDER BY name').all() as { name: string; game: string | null; map: string | null }[];
return rows.map(({ current_json, latest_json, ...row }) => ({ ...row,
currentVersion: current_json ? JSON.parse(current_json) as ScriptVersionRow : null,
latestVersion: latest_json ? JSON.parse(latest_json) as ScriptVersionRow : null,
servers: row.scope_type && row.scope_value ? servers.filter(server => server[row.scope_type!] === row.scope_value).map(server => server.name) : []
${kind ? 'WHERE s.kind=?' : ''} ORDER BY s.name`
)
.all(...(kind ? [kind] : [])) as Array<
ScriptRow & {
current_json: string | null;
latest_json: string | null;
versionCount: number;
runningBetaId: number | null;
}
>;
const servers = db().prepare('SELECT name, game, map FROM servers ORDER BY name').all() as {
name: string;
game: string | null;
map: string | null;
}[];
return rows.map(({ current_json, latest_json, ...row }) => ({
...row,
currentVersion: current_json ? (JSON.parse(current_json) as ScriptVersionRow) : null,
latestVersion: latest_json ? (JSON.parse(latest_json) as ScriptVersionRow) : null,
servers:
row.scope_type && row.scope_value
? servers
.filter((server) => server[row.scope_type!] === row.scope_value)
.map((server) => server.name)
: []
}));
}
export function betaScriptMetadata(ids: number[]) {
const rows = db().prepare(`SELECT bs.beta_id, bs.script_id id, COALESCE(s.name,'(deleted)') name,
const rows = db()
.prepare(
`SELECT bs.beta_id, bs.script_id id, COALESCE(s.name,'(deleted)') name,
COALESCE(s.target_rel_path,'') target_rel_path, COALESCE(s.kind,bs.kind,'script') kind,
v.id version_id, v.version, v.release_version, v.sha256
FROM beta_scripts bs LEFT JOIN scripts s ON s.id=bs.script_id
LEFT JOIN script_versions v ON v.id=bs.script_version_id
WHERE bs.beta_id IN (SELECT value FROM json_each(?)) ORDER BY bs.script_id`).all(JSON.stringify(ids)) as Array<{
beta_id: number; id: number; name: string; target_rel_path: string; kind: ArtifactKind;
version_id: number | null; version: number; release_version: string | null; sha256: string;
}>;
const grouped = new Map<number, Array<{ id: number; name: string; target_rel_path: string; kind: ArtifactKind;
version: { id: number; version: number; release_version: string | null; sha256: string } | null }>>();
WHERE bs.beta_id IN (SELECT value FROM json_each(?)) ORDER BY bs.script_id`
)
.all(JSON.stringify(ids)) as Array<{
beta_id: number;
id: number;
name: string;
target_rel_path: string;
kind: ArtifactKind;
version_id: number | null;
version: number;
release_version: string | null;
sha256: string;
}>;
const grouped = new Map<
number,
Array<{
id: number;
name: string;
target_rel_path: string;
kind: ArtifactKind;
version: {
id: number;
version: number;
release_version: string | null;
sha256: string;
} | null;
}>
>();
for (const { beta_id, version_id, version, release_version, sha256, ...script } of rows) {
const group = grouped.get(beta_id) ?? [];
group.push({ ...script, version: version_id === null ? null : { id: version_id, version, release_version, sha256 } });
group.push({
...script,
version:
version_id === null ? null : { id: version_id, version, release_version, sha256 }
});
grouped.set(beta_id, group);
}
return grouped;
src/lib/server/fleet/lifecycle-evidence.ts
+15 -6
View File
@@ -2,18 +2,27 @@ import { db } from '../db/index.js';
/** Reports from before the latest lifecycle submission describe the old process. */
export function latestGameSubmission(server: string): string | null {
return (db().prepare(`SELECT MAX(at) AS at FROM (
return (
db()
.prepare(
`SELECT MAX(at) AS at FROM (
SELECT MAX(submitted_at) AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`
)
.get(server, server) as { at: string | null }
).at;
}
/** Look up each registered server through the covering submission index. */
export function latestGameSubmissions(): Map<string, string | null> {
const rows = db().prepare(`SELECT server, MAX(at) AS at FROM (
const rows = db()
.prepare(
`SELECT server, MAX(at) AS at FROM (
SELECT s.name AS server, (SELECT MAX(submitted_at) FROM server_deployments
WHERE server=s.name AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')) AS at
FROM servers s
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as
{ server: string; at: string | null }[];
return new Map(rows.map(row => [row.server, row.at]));
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`
)
.all() as { server: string; at: string | null }[];
return new Map(rows.map((row) => [row.server, row.at]));
}
src/lib/server/fleet/logs.ts
+60 -15
View File
@@ -14,7 +14,8 @@ export const KNOWN_LOGS = {
} as const;
export type LogKey = keyof typeof KNOWN_LOGS;
export const gameLogKey = (server: string): LogKey => gameOfServer(server) === 't6' ? 'games_zm' : 'games_sp';
export const gameLogKey = (server: string): LogKey =>
gameOfServer(server) === 't6' ? 'games_zm' : 'games_sp';
export function isLogKey(v: string): v is LogKey {
return Object.prototype.hasOwnProperty.call(KNOWN_LOGS, v);
@@ -45,33 +46,62 @@ export function resolveLogPath(serverName: string, key: LogKey): string | null {
export interface TailResult {
lines: string[];
size: number;
mtime: string; // ISO
mtime: string; // ISO
truncated: boolean; // true when we didn't read from the start of the file
}
export interface LogCursor {
fileId: string; offset: number; size: number; modified: string;
anchorLength: number; anchorHash: string;
fileId: string;
offset: number;
size: number;
modified: string;
anchorLength: number;
anchorHash: string;
}
export interface LogChunk {
contentBase64: string; size: number; mtime: string; truncated: boolean;
cursor: LogCursor; reset: boolean;
contentBase64: string;
size: number;
mtime: string;
truncated: boolean;
cursor: LogCursor;
reset: boolean;
}
/** Bounded byte reads; an anchor before the offset detects copy-truncate and regrowth. */
export async function readLogChunk(path: string, cursor?: LogCursor): Promise<LogChunk | TailResult | null> {
export async function readLogChunk(
path: string,
cursor?: LogCursor
): Promise<LogChunk | TailResult | null> {
const remote = await nativeLogChunk(path, cursor);
if (remote !== undefined) return remote;
let file;
try { file = await open(path, 'r'); }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; throw error; }
try {
file = await open(path, 'r');
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw error;
}
try {
const st = await file.stat();
if (!st.isFile()) return null;
const fileId = `${st.dev}:${st.ino}`, modified = `${st.mtimeMs}:${st.ctimeMs}`, size = st.size;
const fileId = `${st.dev}:${st.ino}`,
modified = `${st.mtimeMs}:${st.ctimeMs}`,
size = st.size;
const mtime = st.mtime.toISOString();
if (cursor?.fileId === fileId && cursor.modified === modified && cursor.size === size && cursor.offset === size) {
return { contentBase64: '', size, mtime, truncated: cursor.offset > 0, cursor, reset: false };
if (
cursor?.fileId === fileId &&
cursor.modified === modified &&
cursor.size === size &&
cursor.offset === size
) {
return {
contentBase64: '',
size,
mtime,
truncated: cursor.offset > 0,
cursor,
reset: false
};
}
const hash = (value: Buffer) => createHash('sha256').update(value).digest('hex');
let reset = !cursor || cursor.fileId !== fileId || cursor.offset > size;
@@ -86,9 +116,24 @@ export async function readLogChunk(path: string, cursor?: LogCursor): Promise<Lo
const offset = start + read.bytesRead;
const anchor = Buffer.alloc(Math.min(256, offset));
await file.read(anchor, 0, anchor.length, offset - anchor.length);
return { contentBase64: content.subarray(0, read.bytesRead).toString('base64'), size, mtime, truncated: start > 0, reset,
cursor: { fileId, modified, size, offset, anchorLength: anchor.length, anchorHash: hash(anchor) } };
} finally { await file.close(); }
return {
contentBase64: content.subarray(0, read.bytesRead).toString('base64'),
size,
mtime,
truncated: start > 0,
reset,
cursor: {
fileId,
modified,
size,
offset,
anchorLength: anchor.length,
anchorHash: hash(anchor)
}
};
} finally {
await file.close();
}
}
const DEFAULT_TAIL_LINES = 200;
src/lib/server/fleet/match-history.ts
+92 -27
View File
@@ -1,6 +1,12 @@
import { db } from '../db/index.js';
import { matchChatTimeline } from './chat.js';
import { MIN_RECORDED_ROUND, type MatchDetail, type MatchPlayer, type MatchHistoryResponse, type ServerMatch } from '../../match-history.js';
import {
MIN_RECORDED_ROUND,
type MatchDetail,
type MatchPlayer,
type MatchHistoryResponse,
type ServerMatch
} from '../../match-history.js';
import { readMatchQuery, type MatchHistoryQuery } from '../../match-history-query.js';
@@ -15,29 +21,69 @@ const matchSummary = `m.id, m.game, m.highest_round AS observedRound, m.server,
MAX(0, ROUND((julianday(m.ended_at) - julianday(m.started_at)) * 86400000)) END AS durationMs,
CASE WHEN m.status = 'completed' THEN m.highest_round END AS finalRound`;
export function serverMatchHistory(server?: string, limit = 25, query: MatchHistoryQuery = readMatchQuery(new URLSearchParams()), now = Date.now()): MatchHistoryResponse {
export function serverMatchHistory(
server?: string,
limit = 25,
query: MatchHistoryQuery = readMatchQuery(new URLSearchParams()),
now = Date.now()
): MatchHistoryResponse {
const handle = db();
const clauses = [`m.highest_round >= ${MIN_RECORDED_ROUND}`];
const parameters: Array<string | number> = [];
if (server !== undefined) { clauses.push('m.server = ?'); parameters.push(server); }
if (query.game) { clauses.push('m.game = ?'); parameters.push(query.game); }
const maps = handle.prepare(`SELECT m.map_key AS key, MAX(m.map_name) AS name, m.game
FROM leaderboard_matches m WHERE ${clauses.join(' AND ')} GROUP BY m.map_key, m.game ORDER BY name COLLATE NOCASE, game`)
if (server !== undefined) {
clauses.push('m.server = ?');
parameters.push(server);
}
if (query.game) {
clauses.push('m.game = ?');
parameters.push(query.game);
}
const maps = handle
.prepare(
`SELECT m.map_key AS key, MAX(m.map_name) AS name, m.game
FROM leaderboard_matches m WHERE ${clauses.join(' AND ')} GROUP BY m.map_key, m.game ORDER BY name COLLATE NOCASE, game`
)
.all(...parameters) as MatchHistoryResponse['maps'];
if (query.map) { clauses.push('m.map_key = ?'); parameters.push(query.map); }
if (query.status) { clauses.push('m.status = ?'); parameters.push(query.status); }
if (query.days) { clauses.push('julianday(m.started_at) >= julianday(?)'); parameters.push(new Date(now - query.days * 86400000).toISOString()); }
if (query.map) {
clauses.push('m.map_key = ?');
parameters.push(query.map);
}
if (query.status) {
clauses.push('m.status = ?');
parameters.push(query.status);
}
if (query.days) {
clauses.push('julianday(m.started_at) >= julianday(?)');
parameters.push(new Date(now - query.days * 86400000).toISOString());
}
// Search readable instance names as well as their stored identifiers. Tokens allow "Town Lobby 1".
for (const token of query.q.toLowerCase().replace(/\blobby\s+(?=\d+\b)/g, '').split(/[\s_]+/).filter(Boolean)) {
clauses.push("instr(lower(m.server || ' ' || m.map_name), ?) > 0"); parameters.push(token);
for (const token of query.q
.toLowerCase()
.replace(/\blobby\s+(?=\d+\b)/g, '')
.split(/[\s_]+/)
.filter(Boolean)) {
clauses.push("instr(lower(m.server || ' ' || m.map_name), ?) > 0");
parameters.push(token);
}
const where = `WHERE ${clauses.join(' AND ')}`;
const total = (handle.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`).get(...parameters) as { total: number }).total;
const total = (
handle
.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`)
.get(...parameters) as { total: number }
).total;
const page = Math.min(query.page, Math.max(1, Math.ceil(total / limit)));
const column = { started: 'startedAt', round: 'observedRound', duration: 'durationMs', players: 'uniquePlayers' }[query.sort];
const column = {
started: 'startedAt',
round: 'observedRound',
duration: 'durationMs',
players: 'uniquePlayers'
}[query.sort];
const direction = query.direction === 'asc' ? 'ASC' : 'DESC';
const matches = handle.prepare(`SELECT ${matchSummary} FROM leaderboard_matches m
${where} ORDER BY ${column} IS NULL ASC, ${column} ${direction}, m.id DESC LIMIT ? OFFSET ?`)
const matches = handle
.prepare(
`SELECT ${matchSummary} FROM leaderboard_matches m
${where} ORDER BY ${column} IS NULL ASC, ${column} ${direction}, m.id DESC LIMIT ? OFFSET ?`
)
.all(...parameters, limit, (page - 1) * limit) as ServerMatch[];
return { matches, total, page, limit, maps };
}
@@ -46,13 +92,20 @@ export function serverMatchHistory(server?: string, limit = 25, query: MatchHist
export function getMatchDetail(id: number): MatchDetail | null {
if (!Number.isSafeInteger(id) || id < 1) return null;
const handle = db();
const match = handle.prepare(`SELECT ${matchSummary}, m.start_observed AS startObserved
const match = handle
.prepare(
`SELECT ${matchSummary}, m.start_observed AS startObserved
FROM leaderboard_matches m WHERE m.id = ? AND m.status = 'completed'
AND m.highest_round >= ${MIN_RECORDED_ROUND}`)
.get(id) as (Omit<MatchDetail, 'players' | 'chat' | 'startObserved'> & { startObserved: number }) | undefined;
AND m.highest_round >= ${MIN_RECORDED_ROUND}`
)
.get(id) as
| (Omit<MatchDetail, 'players' | 'chat' | 'startObserved'> & { startObserved: number })
| undefined;
if (!match) return null;
const players = handle.prepare(`SELECT p.guid,
const players = handle
.prepare(
`SELECT p.guid,
COALESCE((SELECT identity.name FROM leaderboard_players identity
WHERE identity.guid = p.guid AND identity.game IN ('t5', 't6')
ORDER BY identity.last_seen_at DESC, identity.game LIMIT 1),
@@ -76,12 +129,24 @@ export function getMatchDetail(id: number): MatchDetail | null {
AND history.highest_round >= ${MIN_RECORDED_ROUND}
AND recorded.highest_round >= ${MIN_RECORDED_ROUND})) AS canViewProfile
FROM leaderboard_match_players p WHERE p.match_id = ?
ORDER BY hasStats DESC, kills DESC, highestRound DESC, name COLLATE NOCASE, p.guid`)
.all(match.game, id) as Array<Omit<MatchPlayer, 'hasStats' | 'presentAtEnd' | 'canViewProfile'> & {
hasStats: number; presentAtEnd: number; canViewProfile: number;
}>;
return { ...match, chat: matchChatTimeline(id), startObserved: Boolean(match.startObserved), players: players.map(player => ({
...player, hasStats: Boolean(player.hasStats), presentAtEnd: Boolean(player.presentAtEnd),
canViewProfile: Boolean(player.canViewProfile)
})) };
ORDER BY hasStats DESC, kills DESC, highestRound DESC, name COLLATE NOCASE, p.guid`
)
.all(match.game, id) as Array<
Omit<MatchPlayer, 'hasStats' | 'presentAtEnd' | 'canViewProfile'> & {
hasStats: number;
presentAtEnd: number;
canViewProfile: number;
}
>;
return {
...match,
chat: matchChatTimeline(id),
startObserved: Boolean(match.startObserved),
players: players.map((player) => ({
...player,
hasStats: Boolean(player.hasStats),
presentAtEnd: Boolean(player.presentAtEnd),
canViewProfile: Boolean(player.canViewProfile)
}))
};
}
src/lib/server/fleet/metrics.ts
+196 -49
View File
@@ -1,8 +1,18 @@
import { fleetSettings } from '../settings.js';
import { db } from '../db/index.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { parseMetricsQuery, type HeatmapCell, type MapMetrics, type MetricPoint, type MetricsData,
type MetricsDays, type MetricsQuery, type PreviousMetrics, type ServerMetrics, type Utilization } from '../../metrics.js';
import {
parseMetricsQuery,
type HeatmapCell,
type MapMetrics,
type MetricPoint,
type MetricsData,
type MetricsDays,
type MetricsQuery,
type PreviousMetrics,
type ServerMetrics,
type Utilization
} from '../../metrics.js';
export type { MapMetrics, Utilization } from '../../metrics.js';
@@ -17,10 +27,13 @@ export interface MetricSample {
export function recordMetrics(samples: MetricSample[], at = Date.now()): void {
const handle = db();
handle.transaction(() => {
const insert = handle.prepare('INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)');
const insert = handle.prepare(
'INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)'
);
for (const s of samples) insert.run(at, s.server, s.game, s.map, s.players, s.capacity);
const days = fleetSettings().metricsKeepDays;
if (days) handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - days * 86400_000);
if (days)
handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - days * 86400_000);
})();
}
@@ -42,52 +55,120 @@ function addObservation(total: Utilization, players: number | null, hours: numbe
* capped at the freshness limit; longer gaps contribute no observed or unknown time.
* Read the indexed history once and share the intervals across every aggregation. */
export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()): MetricsData {
const query = parseMetricsQuery(new URLSearchParams(typeof input === 'number' ? { days: String(input) }
: { days: String(input.days), game: input.game ?? '', map: input.map ?? '', timeZone: input.timeZone }));
const query = parseMetricsQuery(
new URLSearchParams(
typeof input === 'number'
? { days: String(input) }
: {
days: String(input.days),
game: input.game ?? '',
map: input.map ?? '',
timeZone: input.timeZone
}
)
);
const { days, ...scope } = query;
const since = now - days * 86400_000;
const bucketMs = days === 1 ? 3600_000 : 86400_000;
const handle = db();
// These small grouped results keep options and retained evidence independent
// of the chosen scope/window without transferring a second raw history scan.
const groups = handle.prepare(`SELECT game, map, MIN(at) AS first, MAX(at) AS latest
FROM metric_samples WHERE at <= ? GROUP BY game, map ORDER BY game, map`).all(now) as
{ game: string; map: string; first: number; latest: number }[];
const groups = handle
.prepare(
`SELECT game, map, MIN(at) AS first, MAX(at) AS latest
FROM metric_samples WHERE at <= ? GROUP BY game, map ORDER BY game, map`
)
.all(now) as { game: string; map: string; first: number; latest: number }[];
const matchesScope = (sample: { game: string; map: string }) =>
(scope.game === null || sample.game === scope.game) && (scope.map === null || sample.map === scope.map);
const evidenceBounds = (values: typeof groups) => values.reduce<{ first: number | null; latest: number | null }>((bounds, group) => ({
first: bounds.first === null ? group.first : Math.min(bounds.first, group.first),
latest: bounds.latest === null ? group.latest : Math.max(bounds.latest, group.latest)
}), { first: null, latest: null });
(scope.game === null || sample.game === scope.game) &&
(scope.map === null || sample.map === scope.map);
const evidenceBounds = (values: typeof groups) =>
values.reduce<{ first: number | null; latest: number | null }>(
(bounds, group) => ({
first: bounds.first === null ? group.first : Math.min(bounds.first, group.first),
latest:
bounds.latest === null ? group.latest : Math.max(bounds.latest, group.latest)
}),
{ first: null, latest: null }
);
const globalBounds = evidenceBounds(groups);
const scopedBounds = evidenceBounds(groups.filter(matchesScope));
const registered = new Set((handle.prepare('SELECT name FROM servers WHERE destroyed_at IS NULL').all() as { name: string }[])
.map(server => server.name));
const metadata = { firstRetainedSampleAt: globalBounds.first, latestSampleAt: globalBounds.latest,
registeredServerCount: registered.size, scopeFirstSampleAt: scopedBounds.first, scopeLatestSampleAt: scopedBounds.latest };
const scopeOptions = { games: [...new Set(groups.map(group => group.game))], maps: groups.map(({ game, map }) => ({ game, map })) };
const registered = new Set(
(
handle.prepare('SELECT name FROM servers WHERE destroyed_at IS NULL').all() as {
name: string;
}[]
).map((server) => server.name)
);
const metadata = {
firstRetainedSampleAt: globalBounds.first,
latestSampleAt: globalBounds.latest,
registeredServerCount: registered.size,
scopeFirstSampleAt: scopedBounds.first,
scopeLatestSampleAt: scopedBounds.latest
};
const scopeOptions = {
games: [...new Set(groups.map((group) => group.game))],
maps: groups.map(({ game, map }) => ({ game, map }))
};
const summary = emptyUtilization();
const previous: PreviousMetrics = { since: since - days * 86400_000, until: since, summary: emptyUtilization(),
firstSampleAt: null, lastSampleAt: null, partialHistory: false };
const previous: PreviousMetrics = {
since: since - days * 86400_000,
until: since,
summary: emptyUtilization(),
firstSampleAt: null,
lastSampleAt: null,
partialHistory: false
};
const serverTotals = new Map<string, ServerMetrics>();
const mapTotals = new Map<string, Map<string, MapMetrics>>();
const points: MetricPoint[] = [];
for (let at = Math.floor(since / bucketMs) * bucketMs; at < now; at += bucketMs) {
const end = Math.min(at + bucketMs, now);
const start = Math.max(at, since);
points.push({ at, startAt: start, endAt: end, partial: start !== at || end !== at + bucketMs, ...emptyUtilization() });
points.push({
at,
startAt: start,
endAt: end,
partial: start !== at || end !== at + bucketMs,
...emptyUtilization()
});
}
const cells: HeatmapCell[] = Array.from({ length: 7 * 24 }, (_, index) => ({
weekday: Math.floor(index / 24), hour: index % 24, distinctDates: 0, ...emptyUtilization()
weekday: Math.floor(index / 24),
hour: index % 24,
distinctDates: 0,
...emptyUtilization()
}));
const cellDates = cells.map(() => new Set<string>());
const calendar = new Intl.DateTimeFormat('en-US', { timeZone: scope.timeZone,
year: 'numeric', month: '2-digit', day: '2-digit', weekday: 'short', hour: 'numeric', hourCycle: 'h23' });
const weekdays: Record<string, number> = { Mon: 0, Tue: 1, Wed: 2, Thu: 3, Fri: 4, Sat: 5, Sun: 6 };
const calendar = new Intl.DateTimeFormat('en-US', {
timeZone: scope.timeZone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
weekday: 'short',
hour: 'numeric',
hourCycle: 'h23'
});
const weekdays: Record<string, number> = {
Mon: 0,
Tue: 1,
Wed: 2,
Thu: 3,
Fri: 4,
Sat: 5,
Sun: 6
};
let cachedMinute = NaN;
let localCell = { index: 0, date: '' };
interface BatchCounts { known: number; active: number; idle: number; unknown: number; players: number }
interface BatchCounts {
known: number;
active: number;
idle: number;
unknown: number;
players: number;
}
function addHeatmap(counts: BatchCounts, start: number, end: number): void {
// Samples are bounded by the shared freshness window. Splitting at UTC minute boundaries
// handles local hour/day boundaries, DST, and fractional-hour zones.
@@ -95,8 +176,13 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
for (let cursor = start; cursor < end;) {
const minute = Math.floor(cursor / 60_000);
if (minute !== cachedMinute) {
const parts = Object.fromEntries(calendar.formatToParts(cursor).map(part => [part.type, part.value]));
localCell = { index: weekdays[parts.weekday] * 24 + Number(parts.hour), date: `${parts.year}-${parts.month}-${parts.day}` };
const parts = Object.fromEntries(
calendar.formatToParts(cursor).map((part) => [part.type, part.value])
);
localCell = {
index: weekdays[parts.weekday] * 24 + Number(parts.hour),
date: `${parts.year}-${parts.month}-${parts.day}`
};
cachedMinute = minute;
}
const stop = Math.min(end, (minute + 1) * 60_000);
@@ -120,11 +206,13 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
const samples = scope.game === null ? batch : batch.filter(matchesScope);
if (!samples.length) return;
const end = Math.min(nextAt, at + ONLINE_MAX_MS, now);
const priorHours = Math.max(0, Math.min(end, previous.until) - Math.max(at, previous.since)) / 3600_000;
const priorHours =
Math.max(0, Math.min(end, previous.until) - Math.max(at, previous.since)) / 3600_000;
if (priorHours > 0) {
previous.firstSampleAt ??= at;
previous.lastSampleAt = at;
for (const sample of samples) addObservation(previous.summary, sample.players, priorHours);
for (const sample of samples)
addObservation(previous.summary, sample.players, priorHours);
}
// Preserve current-window zero-duration rows without bringing rows from
// the separate comparison window into the visible server/map tables.
@@ -146,14 +234,27 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
for (const sample of samples) {
let server = serverTotals.get(sample.server);
if (!server) {
server = { server: sample.server, currentRegistered: registered.has(sample.server), ...emptyUtilization() };
server = {
server: sample.server,
currentRegistered: registered.has(sample.server),
...emptyUtilization()
};
serverTotals.set(sample.server, server);
}
let game = mapTotals.get(sample.game);
if (!game) { game = new Map(); mapTotals.set(sample.game, game); }
if (!game) {
game = new Map();
mapTotals.set(sample.game, game);
}
let map = game.get(sample.map);
if (!map) {
map = { game: sample.game, map: sample.map, full: 0, capacityObserved: 0, ...emptyUtilization() };
map = {
game: sample.game,
map: sample.map,
full: 0,
capacityObserved: 0,
...emptyUtilization()
};
game.set(sample.map, map);
}
addObservation(summary, sample.players, hours);
@@ -167,10 +268,14 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
if (sample.players > 0) counts.active++;
else if (sample.players === 0) counts.idle++;
}
const known = sample.players !== null && sample.capacity !== null && sample.capacity > 0;
const known =
sample.players !== null && sample.capacity !== null && sample.capacity > 0;
const full = known && sample.players! >= sample.capacity!;
const previous = pressure.get(map);
pressure.set(map, { known: known && (previous?.known ?? true), full: full && (previous?.full ?? true) });
pressure.set(map, {
known: known && (previous?.known ?? true),
full: full && (previous?.full ?? true)
});
}
for (const [map, state] of pressure) {
if (state.known) {
@@ -181,9 +286,21 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
addHeatmap(counts, start, end);
}
type SampleRow = [at: number, server: string, game: string, map: string, players: number | null, capacity: number | null];
const samples = handle.prepare(`SELECT at, server, game, map, players, capacity FROM metric_samples
WHERE at >= ? AND at <= ? ORDER BY at`).raw().iterate(previous.since - ONLINE_MAX_MS, now) as IterableIterator<SampleRow>;
type SampleRow = [
at: number,
server: string,
game: string,
map: string,
players: number | null,
capacity: number | null
];
const samples = handle
.prepare(
`SELECT at, server, game, map, players, capacity FROM metric_samples
WHERE at >= ? AND at <= ? ORDER BY at`
)
.raw()
.iterate(previous.since - ONLINE_MAX_MS, now) as IterableIterator<SampleRow>;
for (const [at, server, game, map, players, capacity] of samples) {
if (batch.length && at !== batch[0].at) {
aggregateBatch(at);
@@ -193,16 +310,46 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
}
aggregateBatch(now);
const compareText = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0;
const servers = [...serverTotals.values()].sort((a, b) => b.active - a.active || compareText(a.server, b.server));
const maps = [...mapTotals.values()].flatMap(game => [...game.values()])
.sort((a, b) => b.playerHours - a.playerHours || compareText(a.map, b.map) || compareText(a.game, b.game));
previous.partialHistory = scopedBounds.first === null || scopedBounds.first > previous.since || previous.firstSampleAt === null;
for (let index = 0; index < cells.length; index++) cells[index].distinctDates = cellDates[index].size;
const compareText = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);
const servers = [...serverTotals.values()].sort(
(a, b) => b.active - a.active || compareText(a.server, b.server)
);
const maps = [...mapTotals.values()]
.flatMap((game) => [...game.values()])
.sort(
(a, b) =>
b.playerHours - a.playerHours ||
compareText(a.map, b.map) ||
compareText(a.game, b.game)
);
previous.partialHistory =
scopedBounds.first === null ||
scopedBounds.first > previous.since ||
previous.firstSampleAt === null;
for (let index = 0; index < cells.length; index++)
cells[index].distinctDates = cellDates[index].size;
const fullMapHours = maps.reduce((total, map) => total + map.full, 0);
const knownMapHours = maps.reduce((total, map) => total + map.capacityObserved, 0);
const pooledPressure = { value: knownMapHours > 0 ? 100 * fullMapHours / knownMapHours : null,
fullMapHours, knownMapHours, coveredMaps: maps.filter(map => map.capacityObserved > 0).length, totalMaps: maps.length };
return { days, since, now, ...metadata, scope, scopeOptions, summary, servers, maps, points, previous,
heatmap: { timeZone: scope.timeZone, cells }, pooledPressure };
const pooledPressure = {
value: knownMapHours > 0 ? (100 * fullMapHours) / knownMapHours : null,
fullMapHours,
knownMapHours,
coveredMaps: maps.filter((map) => map.capacityObserved > 0).length,
totalMaps: maps.length
};
return {
days,
since,
now,
...metadata,
scope,
scopeOptions,
summary,
servers,
maps,
points,
previous,
heatmap: { timeZone: scope.timeZone, cells },
pooledPressure
};
}
src/lib/server/fleet/monitoring-snapshot.ts
+133 -38
View File
@@ -10,9 +10,22 @@ import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { serverObservation } from './status.js';
import { recoverySettling } from '../provisioning/restarts.js';
const nativeFields = ['id','server','host_id','generation','library_id','image','bind_address','port','public_address','public_port','created_at','retired_at'];
const nativeFields = [
'id',
'server',
'host_id',
'generation',
'library_id',
'image',
'bind_address',
'port',
'public_address',
'public_port',
'created_at',
'retired_at'
];
const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS instance_generation,n.library_id AS game_library_id,
CASE WHEN n.id IS NOT NULL THEN json_object(${nativeFields.map(key => `'${key}',n.${key}`).join(',')}) END AS instance_json,
CASE WHEN n.id IS NOT NULL THEN json_object(${nativeFields.map((key) => `'${key}',n.${key}`).join(',')}) END AS instance_json,
h.inventory AS host_inventory,h.last_heartbeat AS host_heartbeat,
EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id=h.id AND c.revoked_at IS NULL) AS credential_active,
(SELECT json_object('id',d.id,'action',d.action,'state',d.state,'submitted_at',d.submitted_at)
@@ -27,65 +40,147 @@ const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS in
LEFT JOIN ec_hosts h ON h.id=n.host_id AND h.deleted_at IS NULL
LEFT JOIN restart_rollout r ON r.server=s.name LEFT JOIN server_telemetry t ON t.server=s.name`;
interface Row extends ServerRow {
instance_json: string | null; host_inventory: string | null; host_heartbeat: string | null; credential_active: number;
lifecycle_json: string | null; lifecycle_at: string | null; submission: string | null;
restart_at: string | null; restart_healthy: string | null; restart_operation_id: string | null; telemetry_at: string | null; telemetry_json: string | null;
instance_json: string | null;
host_inventory: string | null;
host_heartbeat: string | null;
credential_active: number;
lifecycle_json: string | null;
lifecycle_at: string | null;
submission: string | null;
restart_at: string | null;
restart_healthy: string | null;
restart_operation_id: string | null;
telemetry_at: string | null;
telemetry_json: string | null;
}
interface Record {
server: ServerRow; native: NativeInstance | null;
server: ServerRow;
native: NativeInstance | null;
host: Pick<Host, 'connected' | 'connectionState' | 'inventory'> | null;
lifecycle: { id: string; action: string; state: string; submitted_at: string | null } | null;
lifecycleAt: string | null; submission: string | null; telemetry: GameTelemetry | null;
restartAt: string | null; restartHealthy: string | null; restartOperationId: string | null;
lifecycleAt: string | null;
submission: string | null;
telemetry: GameTelemetry | null;
restartAt: string | null;
restartHealthy: string | null;
restartOperationId: string | null;
}
/** A pass shares its reads; each asynchronous evidence boundary refreshes in one joined query. */
export class MonitoringSnapshot {
private records = new Map<string, Record>();
constructor(serverName?: string) { ensureHostSchema(db()); this.refresh(serverName); }
constructor(serverName?: string) {
ensureHostSchema(db());
this.refresh(serverName);
}
refresh(server?: string): void {
const rows = server === undefined ? db().prepare(sql + ' WHERE s.destroyed_at IS NULL ORDER BY s.name').all() as Row[]
: db().prepare(sql + ' WHERE s.name=? AND s.destroyed_at IS NULL').all(server) as Row[];
if (server === undefined) this.records.clear(); else this.records.delete(server);
const rows =
server === undefined
? (db()
.prepare(sql + ' WHERE s.destroyed_at IS NULL ORDER BY s.name')
.all() as Row[])
: (db()
.prepare(sql + ' WHERE s.name=? AND s.destroyed_at IS NULL')
.all(server) as Row[]);
if (server === undefined) this.records.clear();
else this.records.delete(server);
for (const row of rows) {
const { instance_json,host_inventory,host_heartbeat,credential_active,lifecycle_json,lifecycle_at,submission,
restart_at,restart_healthy,restart_operation_id,telemetry_at,telemetry_json,...fields } = row;
const connected = !!credential_active && !!host_heartbeat && Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
const {
instance_json,
host_inventory,
host_heartbeat,
credential_active,
lifecycle_json,
lifecycle_at,
submission,
restart_at,
restart_healthy,
restart_operation_id,
telemetry_at,
telemetry_json,
...fields
} = row;
const connected =
!!credential_active &&
!!host_heartbeat &&
Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
let telemetry: GameTelemetry | null = null;
if (telemetry_json) try {
const saved = JSON.parse(telemetry_json) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.name) telemetry = { ...event, receivedAt: telemetry_at!, timing: saved.timing };
} catch { /* Invalid telemetry cannot establish health. */ }
this.records.set(row.name, { server: fields, native: instance_json ? JSON.parse(instance_json) : null,
host: host_inventory ? { connected, connectionState: connected ? 'online' : 'offline',
inventory: parseHostInventory(row.host_id!, host_inventory) } : null,
lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null, lifecycleAt: lifecycle_at, submission, telemetry,
restartAt: restart_at, restartHealthy: restart_healthy, restartOperationId: restart_operation_id });
if (telemetry_json)
try {
const saved = JSON.parse(telemetry_json) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.name)
telemetry = { ...event, receivedAt: telemetry_at!, timing: saved.timing };
} catch {
/* Invalid telemetry cannot establish health. */
}
this.records.set(row.name, {
server: fields,
native: instance_json ? JSON.parse(instance_json) : null,
host: host_inventory
? {
connected,
connectionState: connected ? 'online' : 'offline',
inventory: parseHostInventory(row.host_id!, host_inventory)
}
: null,
lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null,
lifecycleAt: lifecycle_at,
submission,
telemetry,
restartAt: restart_at,
restartHealthy: restart_healthy,
restartOperationId: restart_operation_id
});
}
}
get(server: string) { return this.records.get(server); }
servers() { return [...this.records.values()].map(record => record.server); }
get(server: string) {
return this.records.get(server);
}
servers() {
return [...this.records.values()].map((record) => record.server);
}
provisioning(server: string) {
const operation = this.get(server)?.lifecycle;
return !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state);
return (
!!operation &&
['create', 'rebuild', 'start'].includes(operation.action) &&
['queued', 'running', 'awaiting-health'].includes(operation.state)
);
}
settling(server: string) {
const record = this.get(server);
return recoverySettling(record?.restartAt ? { requested_at: record.restartAt,
healthy_at: record.restartHealthy, operation_id: record.restartOperationId } : undefined, record?.lifecycle);
return recoverySettling(
record?.restartAt
? {
requested_at: record.restartAt,
healthy_at: record.restartHealthy,
operation_id: record.restartOperationId
}
: undefined,
record?.lifecycle
);
}
telemetryEvidence(server: string) {
const record = this.get(server), report = record?.telemetry;
const record = this.get(server),
report = record?.telemetry;
if (!report) return null;
const received = Date.parse(report.receivedAt), age = Date.now() - received;
return Number.isFinite(age) && age >= 0 && age <= ONLINE_MAX_MS
&& (!record?.submission || received > Date.parse(record.submission)) ? report : null;
const received = Date.parse(report.receivedAt),
age = Date.now() - received;
return Number.isFinite(age) &&
age >= 0 &&
age <= ONLINE_MAX_MS &&
(!record?.submission || received > Date.parse(record.submission))
? report
: null;
}
observe = (server: ServerRow) => serverObservation(server, {
provisioning: this.provisioning(server.name), settling: this.settling(server.name),
submission: this.get(server.name)?.submission ?? null, telemetry: this.get(server.name)?.telemetry ?? null
});
observe = (server: ServerRow) =>
serverObservation(server, {
provisioning: this.provisioning(server.name),
settling: this.settling(server.name),
submission: this.get(server.name)?.submission ?? null,
telemetry: this.get(server.name)?.telemetry ?? null
});
patch(server: string, fields: Partial<ServerRow>) {
const record = this.get(server);
if (record) record.server = { ...record.server, ...fields };
src/lib/server/fleet/overview.ts
+176 -54
View File
@@ -15,90 +15,212 @@ export function serverOperations(server: string, now = Date.now()): FleetOperati
function readOperations(now: number, server?: string): FleetOperation[] {
const since = new Date(now - 86_400_000).toISOString();
const bindings = { server: server ?? null, since };
const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`;
const scope = (column: string) => (server === undefined ? '' : `AND ${column} = @server`);
const handle = db();
const operations: FleetOperation[] = [];
const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`)
const slots = handle
.prepare(
`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`
)
.all(bindings) as { server: string; requested_at: string; error: string | null }[];
const recovering = new Map(slots.map(slot => [slot.server, slot]));
const recovering = new Map(slots.map((slot) => [slot.server, slot]));
const representedRecovery = new Set<string>();
const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
const campaigns = handle
.prepare(
`SELECT d.id, d.action, d.started_at, ds.server,
ds.restart_state, ds.error, ds.restarted_at, ds.players
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as {
id: string; action: string; started_at: string; server: string; restart_state: string;
error: string | null; restarted_at: string | null; players: number | null;
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`
)
.all(bindings) as {
id: string;
action: string;
started_at: string;
server: string;
restart_state: string;
error: string | null;
restarted_at: string | null;
players: number | null;
}[];
for (const row of campaigns) {
const state = row.restart_state === 'done' ? 'done'
: ['failed', 'manual'].includes(row.restart_state) ? 'attention' : 'active';
const state =
row.restart_state === 'done'
? 'done'
: ['failed', 'manual'].includes(row.restart_state)
? 'attention'
: 'active';
const slot = recovering.get(row.server);
if (row.restarted_at && state !== 'done') representedRecovery.add(row.server);
const label = state === 'done' ? 'Update complete' : state === 'attention' ? 'Update needs attention'
: row.restart_state === 'held' ? 'Held by a beta'
: row.restarted_at ? 'Recovering' : (row.players ?? 0) > 0 ? 'Waiting for an empty lobby' : 'Queued for restart';
operations.push({ key: `campaign:${row.id}:${row.server}`, group: `campaign:${row.id}`,
kind: 'campaign', server: row.server, label, state, startedAt: row.started_at,
detail: state === 'done' ? 'Update applied and restart checks completed' : row.error ?? (row.restarted_at ? slot?.error ?? 'Verifying server health'
: (row.players ?? 0) > 0 ? `${row.players} players at the last check`
: 'Waiting for restart eligibility and available capacity'),
href: `/campaigns/${encodeURIComponent(row.id)}` });
const label =
state === 'done'
? 'Update complete'
: state === 'attention'
? 'Update needs attention'
: row.restart_state === 'held'
? 'Held by a beta'
: row.restarted_at
? 'Recovering'
: (row.players ?? 0) > 0
? 'Waiting for an empty lobby'
: 'Queued for restart';
operations.push({
key: `campaign:${row.id}:${row.server}`,
group: `campaign:${row.id}`,
kind: 'campaign',
server: row.server,
label,
state,
startedAt: row.started_at,
detail:
state === 'done'
? 'Update applied and restart checks completed'
: (row.error ??
(row.restarted_at
? (slot?.error ?? 'Verifying server health')
: (row.players ?? 0) > 0
? `${row.players} players at the last check`
: 'Waiting for restart eligibility and available capacity')),
href: `/campaigns/${encodeURIComponent(row.id)}`
});
}
// Only the newest lifecycle operation can still require action on a server.
// Separate branches use state/rowid indexes for fleet reads and the server
// index for detail history. Completed work is never loaded for the full fleet.
const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
const lifecycle = handle
.prepare(
`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
${server === undefined ? '' : `UNION
${
server === undefined
? ''
: `UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND server = @server
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`}
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`
}
UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`)
.all(bindings) as { id: string; server: string; action: string; state: string; stage: string;
error: string | null; started_at: string }[];
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`
)
.all(bindings) as {
id: string;
server: string;
action: string;
state: string;
stage: string;
error: string | null;
started_at: string;
}[];
for (const row of lifecycle) {
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
const state = ['done', 'warning', 'cancelled'].includes(row.state) ? 'done'
: ['failed', 'interrupted'].includes(row.state) ? 'attention' : 'active';
operations.push({ key: `deployment:${row.id}`, group: `deployment:${row.id}`, kind: 'deployment',
server: row.server, state, startedAt: row.started_at,
activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`, resultState: row.state,
label: state === 'done' ? row.state === 'cancelled' ? 'Removal cancelled' : row.state === 'warning' ? 'Completed with warnings' : 'Complete'
: state === 'attention' ? `${actionLabels[row.action as ProvisioningAction]} needs attention` : row.stage,
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
const state = ['done', 'warning', 'cancelled'].includes(row.state)
? 'done'
: ['failed', 'interrupted'].includes(row.state)
? 'attention'
: 'active';
operations.push({
key: `deployment:${row.id}`,
group: `deployment:${row.id}`,
kind: 'deployment',
server: row.server,
state,
startedAt: row.started_at,
activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`,
resultState: row.state,
label:
state === 'done'
? row.state === 'cancelled'
? 'Removal cancelled'
: row.state === 'warning'
? 'Completed with warnings'
: 'Complete'
: state === 'attention'
? `${actionLabels[row.action as ProvisioningAction]} needs attention`
: row.stage,
detail: row.error ?? row.stage,
href: `/provisioning/${encodeURIComponent(row.id)}`
});
}
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`)
.all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' };
const betas = handle
.prepare(
`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`
)
.all(bindings) as {
id: number;
opened_at: string;
server: string;
stage: string;
error: string | null;
}[];
const stages: Record<string, string> = {
queued: 'Waiting for an earlier test wave',
deploying: 'Deploying beta',
'awaiting-empty': 'Waiting for an empty lobby',
restarting: 'Restarting',
'awaiting-health': 'Recovering',
'awaiting-match': 'Observing a match',
ready: 'Ready for review',
failed: 'Beta needs attention'
};
for (const row of betas) {
if (['restarting', 'awaiting-health'].includes(row.stage)) representedRecovery.add(row.server);
operations.push({ key: `beta:${row.id}:${row.server}`, group: `beta:${row.id}`,
kind: 'beta', server: row.server, startedAt: row.opened_at,
state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
if (['restarting', 'awaiting-health'].includes(row.stage))
representedRecovery.add(row.server);
operations.push({
key: `beta:${row.id}:${row.server}`,
group: `beta:${row.id}`,
kind: 'beta',
server: row.server,
startedAt: row.opened_at,
state: row.stage === 'failed' ? 'attention' : 'active',
label: stages[row.stage] ?? row.stage,
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`,
href: `/betas/${row.id}`
});
}
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
.all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
const maintenance = handle
.prepare(
`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`
)
.all(bindings) as {
id: number;
server: string;
kind: string;
detail: string | null;
requested_at: string;
}[];
for (const job of maintenance) {
operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance',
server: job.server, state: 'attention', startedAt: job.requested_at,
label: job.kind === 'update' ? 'Update verification needs attention' : 'Maintenance needs attention',
detail: job.detail ?? 'Inspect the maintenance result', href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}` });
operations.push({
key: `maintenance:${job.id}`,
group: `maintenance:${job.id}`,
kind: 'maintenance',
server: job.server,
state: 'attention',
startedAt: job.requested_at,
label:
job.kind === 'update'
? 'Update verification needs attention'
: 'Maintenance needs attention',
detail: job.detail ?? 'Inspect the maintenance result',
href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}`
});
}
for (const slot of slots) {
if (representedRecovery.has(slot.server)) continue;
operations.push({ key: `recovery:${slot.server}:${slot.requested_at}`, group: `recovery:${slot.server}`,
kind: 'recovery', server: slot.server, startedAt: slot.requested_at, state: slot.error ? 'attention' : 'active',
label: 'Recovering from restart', detail: slot.error ?? 'Verifying server health',
href: `/servers/${encodeURIComponent(slot.server)}` });
operations.push({
key: `recovery:${slot.server}:${slot.requested_at}`,
group: `recovery:${slot.server}`,
kind: 'recovery',
server: slot.server,
startedAt: slot.requested_at,
state: slot.error ? 'attention' : 'active',
label: 'Recovering from restart',
detail: slot.error ?? 'Verifying server health',
href: `/servers/${encodeURIComponent(slot.server)}`
});
}
return operations.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
}
src/lib/server/fleet/player-capacity.ts
+4 -1
View File
@@ -3,7 +3,10 @@ import { monitoringStatus } from './health-observations.js';
import { MonitoringSnapshot } from './monitoring-snapshot.js';
/** Read only the accepted, current-instance sample; never dispatch a probe. */
export function observedPlayerCapacity(server: string, snapshot: MonitoringSnapshot): PlayerCapacityObservation {
export function observedPlayerCapacity(
server: string,
snapshot: MonitoringSnapshot
): PlayerCapacityObservation {
const game = monitoringStatus(server, snapshot)?.game;
const capacity = game?.result.ok ? game.result.status.capacity : null;
return typeof capacity === 'number' && Number.isSafeInteger(capacity) && capacity > 0
src/lib/server/fleet/player-profile.ts
+70 -23
View File
@@ -2,7 +2,13 @@ import { db } from '../db/index.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
import { getPersonalBestRecords } from './leaderboard-records.js';
import { countsAsGamePlayed } from './leaderboard-participation.js';
import type { LeaderboardOptions, PlayerProfile, PlayerStats, PlayerMapRecord, PlayerMatch } from '../../leaderboard.js';
import type {
LeaderboardOptions,
PlayerProfile,
PlayerStats,
PlayerMapRecord,
PlayerMatch
} from '../../leaderboard.js';
const totals = `MAX(mp.highest_round) AS highestRound, SUM(mp.kills) AS kills,
SUM(mp.rounds_survived) AS roundsSurvived, SUM(mp.headshots) AS headshots,
@@ -12,38 +18,68 @@ const scope = `FROM leaderboard_match_players mp JOIN leaderboard_matches m ON m
WHERE m.game IN ('t5', 't6') AND m.status = 'completed' AND mp.guid = ? AND mp.has_stats = 1
AND m.highest_round >= ${MIN_RECORDED_ROUND} AND mp.highest_round >= ${MIN_RECORDED_ROUND}`;
export function getPlayerProfile(guid: string, requestedOffset = 0,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players'> = {}): PlayerProfile | null {
export function getPlayerProfile(
guid: string,
requestedOffset = 0,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players'> = {}
): PlayerProfile | null {
const handle = db();
// Keep game-specific ingestion identities, but show one identity with the latest reported name.
const player = handle.prepare(`SELECT guid, name, MIN(first_seen_at) OVER () AS firstSeen,
const player = handle
.prepare(
`SELECT guid, name, MIN(first_seen_at) OVER () AS firstSeen,
MAX(last_seen_at) OVER () AS lastSeen FROM leaderboard_players
WHERE game IN ('t5', 't6') AND guid = ? ORDER BY last_seen_at DESC, game LIMIT 1`)
WHERE game IN ('t5', 't6') AND guid = ? ORDER BY last_seen_at DESC, game LIMIT 1`
)
.get(guid) as Pick<PlayerProfile, 'guid' | 'name' | 'firstSeen' | 'lastSeen'> | undefined;
if (!player) return null;
const { matchCount, ...stats } = handle.prepare(`SELECT COUNT(*) AS matchCount, ${totals} ${scope}`)
const { matchCount, ...stats } = handle
.prepare(`SELECT COUNT(*) AS matchCount, ${totals} ${scope}`)
.get(guid) as PlayerStats & { matchCount: number };
if (!matchCount) return null;
const historyParameters: Array<string | number> = [guid];
let historyScope = scope;
if (options.game) { historyScope += ' AND m.game = ?'; historyParameters.push(options.game); }
if (options.map) { historyScope += ' AND m.map_key = ?'; historyParameters.push(options.map); }
if (options.players) { historyScope += ' AND m.start_players = ?'; historyParameters.push(options.players); }
const matchesTotal = options.game || options.map || options.players
? (handle.prepare(`SELECT COUNT(*) AS total ${historyScope}`).get(...historyParameters) as { total: number }).total
: matchCount;
if (options.game) {
historyScope += ' AND m.game = ?';
historyParameters.push(options.game);
}
if (options.map) {
historyScope += ' AND m.map_key = ?';
historyParameters.push(options.map);
}
if (options.players) {
historyScope += ' AND m.start_players = ?';
historyParameters.push(options.players);
}
const matchesTotal =
options.game || options.map || options.players
? (
handle
.prepare(`SELECT COUNT(*) AS total ${historyScope}`)
.get(...historyParameters) as { total: number }
).total
: matchCount;
const limit = 25;
const lastOffset = Math.max(0, Math.floor((matchesTotal - 1) / limit) * limit);
const offset = Number.isSafeInteger(requestedOffset) ? Math.min(lastOffset, Math.max(0, requestedOffset)) : 0;
const mapStats = handle.prepare(`SELECT m.game, m.map_key AS key, m.map_name AS name,
const offset = Number.isSafeInteger(requestedOffset)
? Math.min(lastOffset, Math.max(0, requestedOffset))
: 0;
const mapStats = handle
.prepare(
`SELECT m.game, m.map_key AS key, m.map_name AS name,
MAX(mp.last_seen_at) AS lastSeen, ${totals} ${scope}
GROUP BY m.game, m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, m.game, key`)
GROUP BY m.game, m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, m.game, key`
)
.all(guid) as Array<Omit<PlayerMapRecord, 'records' | 'ranks'>>;
const [records, ...mapRecords] = getPersonalBestRecords(handle, [{ guid, ...stats },
...mapStats.map(map => ({ guid, ...map, map: map.key }))]);
const [records, ...mapRecords] = getPersonalBestRecords(handle, [
{ guid, ...stats },
...mapStats.map((map) => ({ guid, ...map, map: map.key }))
]);
// Rank all three bests in one pass over this player's maps. Filter the GUID
// only after ranking competitors so ties and skipped ranks match the leaderboard.
const mapRanks = handle.prepare(`WITH totals AS (
const mapRanks = handle
.prepare(
`WITH totals AS (
SELECT requested.key AS requestId, mp.guid,
MAX(mp.highest_round) AS highestRound, MAX(mp.kills) AS bestKills, MAX(mp.best_score) AS bestScore
FROM json_each(?) requested
@@ -60,14 +96,25 @@ export function getPlayerProfile(guid: string, requestedOffset = 0,
RANK() OVER (PARTITION BY requestId ORDER BY bestKills DESC) AS bestKills,
RANK() OVER (PARTITION BY requestId ORDER BY bestScore DESC) AS bestScore
FROM totals
) SELECT highestRound, bestKills, bestScore FROM ranked WHERE guid = ? ORDER BY requestId`)
.all(JSON.stringify(mapStats.map(({ game, key }) => ({ game, key }))), guid) as PlayerMapRecord['ranks'][];
const maps = mapStats.map((map, index) => ({ ...map, records: mapRecords[index], ranks: mapRanks[index] }));
const matches = handle.prepare(`SELECT m.id, m.game, m.server, m.server_deleted_at, m.map_name AS mapName,
) SELECT highestRound, bestKills, bestScore FROM ranked WHERE guid = ? ORDER BY requestId`
)
.all(
JSON.stringify(mapStats.map(({ game, key }) => ({ game, key }))),
guid
) as PlayerMapRecord['ranks'][];
const maps = mapStats.map((map, index) => ({
...map,
records: mapRecords[index],
ranks: mapRanks[index]
}));
const matches = handle
.prepare(
`SELECT m.id, m.game, m.server, m.server_deleted_at, m.map_name AS mapName,
m.started_at AS startedAt, mp.last_seen_at AS lastSeen, m.status, mp.completed,
mp.highest_round AS highestRound, mp.rounds_survived AS roundsSurvived,
mp.kills, mp.headshots, mp.revives, mp.downs, mp.best_score AS bestScore
${historyScope} ORDER BY mp.last_seen_at DESC, m.id DESC LIMIT ? OFFSET ?`)
${historyScope} ORDER BY mp.last_seen_at DESC, m.id DESC LIMIT ? OFFSET ?`
)
.all(...historyParameters, limit, offset) as PlayerMatch[];
return { ...player, stats, records, maps, matches, matchesTotal, offset, limit };
}
src/lib/server/fleet/plugin-releases.ts
+110 -25
View File
@@ -1,13 +1,29 @@
import { createHash } from 'node:crypto';
import { db } from '../db/index.js';
import { createScript, getScript, getVersion, uploadNewVersion, type ScopeType } from './scripts.js';
import {
createScript,
getScript,
getVersion,
uploadNewVersion,
type ScopeType
} from './scripts.js';
import { findRunningBetaForScript } from './beta.js';
import { downloadGithubAsset, getGithubRelease, GithubReleaseError } from '../integrations/github-releases.js';
import {
downloadGithubAsset,
getGithubRelease,
GithubReleaseError
} from '../integrations/github-releases.js';
import type { PluginSource, PluginUpdate } from '$lib/plugin-github.js';
export function pluginSource(pluginId: number): PluginSource | null {
return db().prepare(`SELECT p.* FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
WHERE v.script_id = ? ORDER BY v.version DESC LIMIT 1`).get(pluginId) as PluginSource | undefined ?? null;
return (
(db()
.prepare(
`SELECT p.* FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
WHERE v.script_id = ? ORDER BY v.version DESC LIMIT 1`
)
.get(pluginId) as PluginSource | undefined) ?? null
);
}
export async function checkPluginUpdate(pluginId: number): Promise<PluginUpdate> {
@@ -19,13 +35,27 @@ export async function checkPluginUpdate(pluginId: number): Promise<PluginUpdate>
try {
const release = await getGithubRelease(`https://github.com/${source.repository}`);
// A latest-stable release may already be in history when a newer prerelease is imported.
if (release.id !== source.release_id && Date.parse(release.publishedAt) <= Date.parse(source.published_at)) {
if (
release.id !== source.release_id &&
Date.parse(release.publishedAt) <= Date.parse(source.published_at)
) {
return { ...base, release, status: 'ahead' };
}
const imported = db().prepare(`SELECT p.version_id FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
const imported = db()
.prepare(
`SELECT p.version_id FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
WHERE v.script_id = ? AND lower(p.repository) = lower(?) AND p.release_id = ? AND p.asset_name = ?
ORDER BY v.version DESC LIMIT 1`).get(pluginId, source.repository, release.id, source.asset_name) as { version_id: number } | undefined;
if (imported) return { ...base, release, status: imported.version_id === plugin.current_version_id ? 'current' : 'imported', versionId: imported.version_id };
ORDER BY v.version DESC LIMIT 1`
)
.get(pluginId, source.repository, release.id, source.asset_name) as
{ version_id: number } | undefined;
if (imported)
return {
...base,
release,
status: imported.version_id === plugin.current_version_id ? 'current' : 'imported',
versionId: imported.version_id
};
return { ...base, release, status: 'available' };
} catch (error) {
return { ...base, release: null, status: 'error', error: (error as Error).message };
@@ -42,44 +72,99 @@ export interface ImportPluginReleaseInput {
}
export async function importPluginRelease(input: ImportPluginReleaseInput) {
if (typeof input.url !== 'string' || !Number.isSafeInteger(input.assetId) || input.assetId < 1) {
if (
typeof input.url !== 'string' ||
!Number.isSafeInteger(input.assetId) ||
input.assetId < 1
) {
throw new GithubReleaseError('A release URL and DLL asset selection are required.');
}
if (input.pluginId !== undefined && (!Number.isSafeInteger(input.pluginId) || input.pluginId < 1)) {
if (
input.pluginId !== undefined &&
(!Number.isSafeInteger(input.pluginId) || input.pluginId < 1)
) {
throw new GithubReleaseError('Invalid plugin id.');
}
const requirePlugin = () => {
const plugin = input.pluginId === undefined ? undefined : getScript(input.pluginId);
if (input.pluginId !== undefined && (!plugin || plugin.kind !== 'plugin')) throw new GithubReleaseError('Plugin not found.', 404);
if (plugin && findRunningBetaForScript(plugin.id)) throw new GithubReleaseError('Finish the running beta before importing another release.', 409);
if (input.pluginId !== undefined && (!plugin || plugin.kind !== 'plugin'))
throw new GithubReleaseError('Plugin not found.', 404);
if (plugin && findRunningBetaForScript(plugin.id))
throw new GithubReleaseError(
'Finish the running beta before importing another release.',
409
);
return plugin;
};
requirePlugin();
if (input.pluginId === undefined && (typeof input.name !== 'string' || !input.name.trim())) throw new GithubReleaseError('A plugin name is required.');
if (input.pluginId === undefined && (typeof input.name !== 'string' || !input.name.trim()))
throw new GithubReleaseError('A plugin name is required.');
// Refetch the exact reviewed release: asset names, notes, tags and digests are never trusted from the client.
const release = await getGithubRelease(input.url, true);
const asset = release.assets.find(a => a.id === input.assetId);
if (!asset) throw new GithubReleaseError('That DLL is no longer available in this release. Preview it again.');
const asset = release.assets.find((a) => a.id === input.assetId);
if (!asset)
throw new GithubReleaseError(
'That DLL is no longer available in this release. Preview it again.'
);
const content = await downloadGithubAsset(release, asset);
const sha = createHash('sha256').update(content).digest('hex');
return db().transaction(() => {
const plugin = requirePlugin();
if (plugin) {
const prior = db().prepare(`SELECT v.id FROM script_versions v JOIN plugin_imports p ON p.version_id = v.id
const prior = db()
.prepare(
`SELECT v.id FROM script_versions v JOIN plugin_imports p ON p.version_id = v.id
WHERE v.script_id = ? AND lower(p.repository) = lower(?) AND p.release_id = ? AND p.asset_id = ? AND v.sha256 = ?
ORDER BY v.version DESC LIMIT 1`).get(plugin.id, release.repository, release.id, asset.id, sha) as { id: number } | undefined;
ORDER BY v.version DESC LIMIT 1`
)
.get(plugin.id, release.repository, release.id, asset.id, sha) as
{ id: number } | undefined;
if (prior) return result(plugin.id, getVersion(prior.id)!, true);
}
const created = plugin ? { script: plugin, version: uploadNewVersion(plugin.id, content, release.notes, release.tag) }
: createScript({ name: input.name!.trim(), kind: 'plugin', targetRelPath: asset.name, content,
releaseVersion: release.tag, notes: release.notes, scopeType: input.scopeType, scopeValue: input.scopeValue });
db().prepare(`INSERT INTO plugin_imports (version_id, repository, release_id, release_url, asset_id, asset_name, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(created.version.id, release.repository, release.id, release.url, asset.id, asset.name, release.publishedAt);
const created = plugin
? {
script: plugin,
version: uploadNewVersion(plugin.id, content, release.notes, release.tag)
}
: createScript({
name: input.name!.trim(),
kind: 'plugin',
targetRelPath: asset.name,
content,
releaseVersion: release.tag,
notes: release.notes,
scopeType: input.scopeType,
scopeValue: input.scopeValue
});
db()
.prepare(
`INSERT INTO plugin_imports (version_id, repository, release_id, release_url, asset_id, asset_name, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`
)
.run(
created.version.id,
release.repository,
release.id,
release.url,
asset.id,
asset.name,
release.publishedAt
);
return result(created.script.id, created.version, false);
})();
}
function result(pluginId: number, version: NonNullable<ReturnType<typeof getVersion>>, alreadyImported: boolean) {
return { pluginId, versionId: version.id, releaseVersion: version.release_version, sha256: version.sha256,
alreadyImported, betaUrl: `/betas?pluginId=${pluginId}&versionId=${version.id}` };
function result(
pluginId: number,
version: NonNullable<ReturnType<typeof getVersion>>,
alreadyImported: boolean
) {
return {
pluginId,
versionId: version.id,
releaseVersion: version.release_version,
sha256: version.sha256,
alreadyImported,
betaUrl: `/betas?pluginId=${pluginId}&versionId=${version.id}`
};
}
src/lib/server/fleet/plutonium.ts
+1 -1
View File
@@ -1,2 +1,2 @@
// Compatibility for existing jobs and API clients.
export * from "../maintenance/runtime.js";
export * from '../maintenance/runtime.js';
src/lib/server/fleet/promoted-artifacts.ts
+12 -4
View File
@@ -1,10 +1,18 @@
import { getVersion, getVersionContent, getVersionMetadata, listScripts } from './scripts.js';
function selectedArtifacts<Version>(game: string | null, map: string | null, readVersion: (id: number) => Version | undefined) {
function selectedArtifacts<Version>(
game: string | null,
map: string | null,
readVersion: (id: number) => Version | undefined
) {
return (['script', 'plugin'] as const).flatMap((kind) =>
listScripts(kind).flatMap((script) => {
const scopeMatches = script.scope_type === 'game' ? script.scope_value === game
: script.scope_type === 'map' ? script.scope_value === map : false;
const scopeMatches =
script.scope_type === 'game'
? script.scope_value === game
: script.scope_type === 'map'
? script.scope_value === map
: false;
if (!scopeMatches || !script.current_version_id) return [];
const version = readVersion(script.current_version_id);
return version ? [{ kind, script, version }] : [];
@@ -14,7 +22,7 @@ function selectedArtifacts<Version>(game: string | null, map: string | null, rea
/** Published file artifacts shared by provisioning and drift reconciliation. */
export function promotedArtifacts(game: string | null, map: string | null) {
return selectedArtifacts(game, map, id => {
return selectedArtifacts(game, map, (id) => {
const version = getVersion(id);
return version ? { ...version, content: getVersionContent(id) } : undefined;
});
src/lib/server/fleet/provisioning.ts
+25 -8
View File
@@ -2,17 +2,34 @@ import { db } from '../db/index.js';
/** Maintenance must leave pending lifecycle work and retained recovery to Provisioning. */
export function hasActiveProvisioning(server: string, exceptOperationId = ''): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments
return (
!!db()
.prepare(
`SELECT 1 FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL AND id != ?
AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1`).get(server, exceptOperationId)
|| !!db().prepare(`SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL
AND (operation_id IS NULL OR operation_id != ?)`).get(server, exceptOperationId);
AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1`
)
.get(server, exceptOperationId) ||
!!db()
.prepare(
`SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL
AND (operation_id IS NULL OR operation_id != ?)`
)
.get(server, exceptOperationId)
);
}
/** Only the latest lifecycle operation can describe the current server. */
export function isProvisioning(server: string): boolean {
const operation = db().prepare(`SELECT action, state FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as
{ action: string; state: string } | undefined;
return !!operation && ['create', 'rebuild', 'start'].includes(operation.action) && ['queued', 'running', 'awaiting-health'].includes(operation.state);
const operation = db()
.prepare(
`SELECT action, state FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`
)
.get(server) as { action: string; state: string } | undefined;
return (
!!operation &&
['create', 'rebuild', 'start'].includes(operation.action) &&
['queued', 'running', 'awaiting-health'].includes(operation.state)
);
}
src/lib/server/fleet/rollback.ts
+148 -50
View File
@@ -3,7 +3,12 @@ import type { ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
import { getVersion } from './scripts.js';
import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js';
import {
cancelStagedFiles,
isFileApplied,
queueFileChange,
stagedFileChanges
} from './staged-files.js';
// Roll a promoted deployment back to whatever versions were current before
// the promote landed. Reads deployment_scripts as the source of truth for
@@ -12,7 +17,15 @@ import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges }
// pre-multi-script legacy rows.
export class RollbackError extends Error {
constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back' | 'snapshot-pending') {
constructor(
message: string,
public readonly kind:
| 'history-expired'
| 'no-such-deployment'
| 'not-promote'
| 'already-rolled-back'
| 'snapshot-pending'
) {
super(message);
}
}
@@ -58,31 +71,53 @@ function nowIso(): string {
}
function rollbackDeploymentId(promoteId: string): string {
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
const stamp = new Date()
.toISOString()
.replace(/[-:]/g, '')
.replace(/\.\d+Z$/, 'Z');
return `dep-${stamp}-rollback-${promoteId}`;
}
export interface RollbackDeploymentResult {
deploymentId: string; // the new rollback deployment
reverses: string; // the promote deployment id
scriptIds: number[]; // scripts whose current_version_id got reverted
deploymentId: string; // the new rollback deployment
reverses: string; // the promote deployment id
scriptIds: number[]; // scripts whose current_version_id got reverted
affectedServers: string[];
}
export async function rollbackPromoteDeployment(promoteId: string): Promise<RollbackDeploymentResult> {
export async function rollbackPromoteDeployment(
promoteId: string
): Promise<RollbackDeploymentResult> {
const dep = db()
.prepare(`SELECT id, action, script_id, script_version_id, prior_version_id, rolled_back_by, beta_id
FROM deployments WHERE id = ?`)
.prepare(
`SELECT id, action, script_id, script_version_id, prior_version_id, rolled_back_by, beta_id
FROM deployments WHERE id = ?`
)
.get(promoteId) as DeploymentRow | undefined;
if (!dep) throw new RollbackError(`no such deployment ${promoteId}`, 'no-such-deployment');
if (db().prepare("SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?").get(promoteId)) {
throw new RollbackError('Rollback file contents have expired under the retention policy.', 'history-expired');
if (
db()
.prepare(
"SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?"
)
.get(promoteId)
) {
throw new RollbackError(
'Rollback file contents have expired under the retention policy.',
'history-expired'
);
}
if (dep.action !== 'promote') {
throw new RollbackError(`deployment ${promoteId} is action='${dep.action}', not 'promote'`, 'not-promote');
throw new RollbackError(
`deployment ${promoteId} is action='${dep.action}', not 'promote'`,
'not-promote'
);
}
if (dep.rolled_back_by) {
throw new RollbackError(`deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`, 'already-rolled-back');
throw new RollbackError(
`deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`,
'already-rolled-back'
);
}
// Source of truth for which scripts (and prior versions) this promote
@@ -92,13 +127,15 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
.prepare(`SELECT * FROM deployment_scripts WHERE deployment_id = ? ORDER BY script_id`)
.all(promoteId) as DepScriptRow[];
if (scripts.length === 0 && dep.script_id !== null && dep.script_version_id !== null) {
scripts = [{
deployment_id: promoteId,
script_id: dep.script_id,
script_version_id: dep.script_version_id,
prior_version_id: dep.prior_version_id,
kind: 'script'
}];
scripts = [
{
deployment_id: promoteId,
script_id: dep.script_id,
script_version_id: dep.script_version_id,
prior_version_id: dep.prior_version_id,
kind: 'script'
}
];
}
const fileRows = db()
@@ -132,17 +169,29 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
const staged = stagedFileChanges('campaign', promoteId);
const scriptForFile = (file: FileRow): DepScriptRow | undefined => {
const candidates = scriptsByTarget.get(file.rel_path) ?? [];
const versionId = staged.find(row => row.deployment_file_id === file.id)?.script_version_id;
if (versionId) return candidates.find(script => script.script_version_id === versionId);
const versionId = staged.find(
(row) => row.deployment_file_id === file.id
)?.script_version_id;
if (versionId) return candidates.find((script) => script.script_version_id === versionId);
if (dep.beta_id) {
const participant = candidates.find(script => db().prepare(`SELECT 1 FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`).get(dep.beta_id, file.server, script.script_id));
const participant = candidates.find((script) =>
db()
.prepare(
`SELECT 1 FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`
)
.get(dep.beta_id, file.server, script.script_id)
);
if (participant) return participant;
}
return candidates.find(script => getVersion(script.script_version_id)?.sha256 === file.sha256_after) ?? candidates[0];
return (
candidates.find(
(script) => getVersion(script.script_version_id)?.sha256 === file.sha256_after
) ?? candidates[0]
);
};
const kindForFile = (file: FileRow): ArtifactKind => {
const stagedKind = staged.find(row => row.deployment_file_id === file.id)?.kind;
const stagedKind = staged.find((row) => row.deployment_file_id === file.id)?.kind;
if (stagedKind) return stagedKind;
const owner = scriptForFile(file);
if (owner) return owner.kind ?? kindByScriptId.get(owner.script_id) ?? 'script';
@@ -164,11 +213,23 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
// must not restore or remove files that it never installed. Rows from
// older releases have no staging record and retain their audit meaning.
const kind = kindForFile(f);
const preservesBetaSnapshot = staged.some(row => row.server === f.server && row.kind === kind
&& row.rel_path === f.rel_path && row.preserve_before_image);
if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue;
const preservesBetaSnapshot = staged.some(
(row) =>
row.server === f.server &&
row.kind === kind &&
row.rel_path === f.rel_path &&
row.preserve_before_image
);
if (
!preservesBetaSnapshot &&
isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false
)
continue;
if (f.snapshot_pending) {
throw new RollbackError(`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`, 'snapshot-pending');
throw new RollbackError(
`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`,
'snapshot-pending'
);
}
const arr = perServer.get(f.server);
if (arr) arr.push(f);
@@ -176,8 +237,13 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
}
const affectedServers: string[] = [];
const active = (server: string) => !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server);
const active = (server: string) =>
!!db()
.prepare(
`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`
)
.get(promoteId, server);
// Write the new rollback deployment + mark the original.
const insertDep = db().prepare(
@@ -204,8 +270,12 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
cancelStagedFiles('campaign', promoteId);
// Withdrawing this source leaves a shared restart available to other
// campaigns. Provisioning cancels an operation with no eligible sources.
db().prepare(`UPDATE deployment_servers SET restart_state = 'skipped', error = NULL
WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).run(promoteId);
db()
.prepare(
`UPDATE deployment_servers SET restart_state = 'skipped', error = NULL
WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`
)
.run(promoteId);
insertDep.run(newDepId, startedAt);
// Mirror the promote's script list — swapped: the "current" version
// after rollback is the promote's prior; the "prior" of the rollback
@@ -221,16 +291,20 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
}
for (const [server, files] of perServer) {
if (!active(server)) continue;
if (files.some(file => file.verb !== 'added' && file.content_before === null)) {
insertDepServer.run(newDepId, server, 0, 'failed', 'Original file snapshot is missing; cannot restore safely');
if (files.some((file) => file.verb !== 'added' && file.content_before === null)) {
insertDepServer.run(
newDepId,
server,
0,
'failed',
'Original file snapshot is missing; cannot restore safely'
);
continue;
}
insertDepServer.run(newDepId, server, files.length, 'pending', null);
for (const f of files) {
const invertedVerb =
f.verb === 'added' ? 'removed'
: f.verb === 'removed' ? 'added'
: 'updated';
f.verb === 'added' ? 'removed' : f.verb === 'removed' ? 'added' : 'updated';
const inserted = insertDepFile.run(
newDepId,
server,
@@ -242,27 +316,51 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
putFileBlob(f.content_before)
);
const owningScript = scriptForFile(f);
const betaPrior = dep.beta_id && owningScript ? db().prepare(`SELECT prior_version_id FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`).get(dep.beta_id, server, owningScript.script_id) as
{ prior_version_id: number | null } | undefined : undefined;
const priorVersionId = betaPrior ? betaPrior.prior_version_id : owningScript?.prior_version_id;
const betaPrior =
dep.beta_id && owningScript
? (db()
.prepare(
`SELECT prior_version_id FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`
)
.get(dep.beta_id, server, owningScript.script_id) as
{ prior_version_id: number | null } | undefined)
: undefined;
const priorVersionId = betaPrior
? betaPrior.prior_version_id
: owningScript?.prior_version_id;
const priorVersion = priorVersionId ? getVersion(priorVersionId) : null;
queueFileChange({ sourceKind: 'campaign', sourceId: newDepId, server,
kind: kindForFile(f), relPath: f.rel_path,
queueFileChange({
sourceKind: 'campaign',
sourceId: newDepId,
server,
kind: kindForFile(f),
relPath: f.rel_path,
content: f.verb === 'added' ? null : f.content_before,
scriptVersionId: f.verb === 'updated' ? priorVersion?.id ?? null : null,
inventorySource: `rollback:${newDepId}`, deploymentFileId: Number(inserted.lastInsertRowid) });
scriptVersionId: f.verb === 'updated' ? (priorVersion?.id ?? null) : null,
inventorySource: `rollback:${newDepId}`,
deploymentFileId: Number(inserted.lastInsertRowid)
});
}
affectedServers.push(server);
}
db().prepare(`UPDATE deployments SET rolled_back_by = ? WHERE id = ?`).run(newDepId, promoteId);
db()
.prepare(`UPDATE deployments SET rolled_back_by = ? WHERE id = ?`)
.run(newDepId, promoteId);
// Restore each affected script's current_version_id.
for (const s of scripts) {
db().prepare(`UPDATE scripts SET current_version_id = ? WHERE id = ?`).run(s.prior_version_id, s.script_id);
db()
.prepare(`UPDATE scripts SET current_version_id = ? WHERE id = ?`)
.run(s.prior_version_id, s.script_id);
}
})();
emit({ type: 'deployment-opened', deploymentId: newDepId, action: 'rollback', servers: affectedServers });
emit({
type: 'deployment-opened',
deploymentId: newDepId,
action: 'rollback',
servers: affectedServers
});
return {
deploymentId: newDepId,
src/lib/server/fleet/runtime-config.ts
+6 -1
View File
@@ -19,7 +19,12 @@ export function applyRuntimePassword(text: string, password?: string | null): st
}
if (!matched) {
if (lines.at(-1) === '') lines.pop();
lines.push('', '// Added by Plutainer from PLUTAINER_RCON_PASSWORD.', `set rcon_password ${replacement}`, '');
lines.push(
'',
'// Added by Plutainer from PLUTAINER_RCON_PASSWORD.',
`set rcon_password ${replacement}`,
''
);
changed = true;
}
return changed ? lines.join('\n') : text;
src/lib/server/fleet/scripts.ts
+137 -35
View File
@@ -5,7 +5,12 @@ import { configFileForGame, type ArtifactKind } from '../config.js';
export type { ArtifactKind };
export const ARTIFACT_KINDS: ReadonlyArray<ArtifactKind> = ['script', 'config', 'plugin', 'baseline'];
export const ARTIFACT_KINDS: ReadonlyArray<ArtifactKind> = [
'script',
'config',
'plugin',
'baseline'
];
export function isArtifactKind(v: unknown): v is ArtifactKind {
return typeof v === 'string' && (ARTIFACT_KINDS as readonly string[]).includes(v);
@@ -78,8 +83,12 @@ function sha256(buf: Buffer): string {
function normalizeReleaseVersion(value: string | null | undefined): string | null {
if (value == null) return null;
// eslint-disable-next-line no-control-regex -- Reject control characters in release versions.
if (typeof value !== 'string' || value.length > 80 || /[\u0000-\u001f\u007f]/.test(value)) {
throw new ScriptError('release version must be a single line of at most 80 characters', 'invalid-release-version');
throw new ScriptError(
'release version must be a single line of at most 80 characters',
'invalid-release-version'
);
}
return value.trim() || null;
}
@@ -94,7 +103,9 @@ export function getScriptByName(name: string): ScriptRow | undefined {
export function listScripts(kind?: ArtifactKind): ScriptRow[] {
if (kind) {
return db().prepare('SELECT * FROM scripts WHERE kind = ? ORDER BY name').all(kind) as ScriptRow[];
return db()
.prepare('SELECT * FROM scripts WHERE kind = ? ORDER BY name')
.all(kind) as ScriptRow[];
}
return db().prepare('SELECT * FROM scripts ORDER BY name').all() as ScriptRow[];
}
@@ -104,7 +115,10 @@ export function listScripts(kind?: ArtifactKind): ScriptRow[] {
// gated through the beta pipeline. `includeUntested` returns the baseline row
// even without a promoted version (used at beta-open time when the baseline is
// being introduced in the same beta as the overlays).
export function getBaselineForScope(scopeType: ScopeType, scopeValue: string): ScriptRow | undefined {
export function getBaselineForScope(
scopeType: ScopeType,
scopeValue: string
): ScriptRow | undefined {
return db()
.prepare(
`SELECT * FROM scripts
@@ -137,40 +151,49 @@ export function listPromotedOverlaysForScope(
return rows.filter((r) => !skip.has(r.id));
}
export const VERSION_COLUMNS = 'id, script_id, version, release_version, sha256, size, notes, state, kind, uploaded_at';
export const VERSION_COLUMNS =
'id, script_id, version, release_version, sha256, size, notes, state, kind, uploaded_at';
export function getVersionContent(id: number): Buffer {
const row = db().prepare(`SELECT ${blobContent('content_hash')} AS content FROM script_versions WHERE id = ?`).get(id) as { content: Buffer } | undefined;
if (!row?.content) throw new ScriptError(`version content is unavailable (id=${id})`, 'no-such-version');
const row = db()
.prepare(
`SELECT ${blobContent('content_hash')} AS content FROM script_versions WHERE id = ?`
)
.get(id) as { content: Buffer } | undefined;
if (!row?.content)
throw new ScriptError(`version content is unavailable (id=${id})`, 'no-such-version');
return row.content;
}
export function listVersions(scriptId: number): ScriptVersionRow[] {
return db()
.prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC`)
.prepare(
`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC`
)
.all(scriptId) as ScriptVersionRow[];
}
export function getVersion(id: number): ScriptVersionRow | undefined {
return db().prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE id = ?`).get(id) as
| ScriptVersionRow
| undefined;
ScriptVersionRow | undefined;
}
export type ScriptVersionMetadata = Pick<ScriptVersionRow, 'id' | 'version' | 'sha256' | 'size'>;
/** Planning a launch never needs to load potentially large script or plugin bytes. */
export function getVersionMetadata(id: number): ScriptVersionMetadata | undefined {
return db().prepare('SELECT id, version, sha256, size FROM script_versions WHERE id = ?').get(id) as
| ScriptVersionMetadata
| undefined;
return db()
.prepare('SELECT id, version, sha256, size FROM script_versions WHERE id = ?')
.get(id) as ScriptVersionMetadata | undefined;
}
// Latest version is 'max(version)' — used everywhere we want the freshest
// upload, whether or not it has been promoted (current_version_id).
export function getLatestVersion(scriptId: number): ScriptVersionRow | undefined {
return db()
.prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC LIMIT 1`)
.prepare(
`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC LIMIT 1`
)
.get(scriptId) as ScriptVersionRow | undefined;
}
@@ -191,7 +214,10 @@ export interface CreateScriptInput {
// Insert script + version 1 in one transaction. Scope may be set at creation
// because the script has no current_version_id yet (nothing to gate).
export function createScript(input: CreateScriptInput): { script: ScriptRow; version: ScriptVersionRow } {
export function createScript(input: CreateScriptInput): {
script: ScriptRow;
version: ScriptVersionRow;
} {
if (!input.name.trim()) throw new ScriptError('script name is required', 'name-empty');
const kind: ArtifactKind = input.kind ?? 'script';
if (!isArtifactKind(kind)) throw new ScriptError(`invalid kind: ${input.kind}`, 'invalid-kind');
@@ -200,15 +226,23 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
// the same on-disk cfg file, so the operator never picks it.
let targetRelPath = input.targetRelPath?.trim() ?? '';
if (!targetRelPath && (kind === 'config' || kind === 'baseline')) {
targetRelPath = configFileForGame(input.scopeType === 'game' ? input.scopeValue ?? null
: input.scopeValue?.startsWith('zm_') ? 't6' : 't5');
targetRelPath = configFileForGame(
input.scopeType === 'game'
? (input.scopeValue ?? null)
: input.scopeValue?.startsWith('zm_')
? 't6'
: 't5'
);
}
if (!targetRelPath) throw new ScriptError('target rel path is required', 'path-empty');
// Kind-specific validation.
if (kind === 'baseline') {
if (!input.scopeType || !input.scopeValue) {
throw new ScriptError('baselines require a scope (game or map)', 'baseline-requires-scope');
throw new ScriptError(
'baselines require a scope (game or map)',
'baseline-requires-scope'
);
}
if (getBaselineForScope(input.scopeType, input.scopeValue)) {
throw new ScriptError(
@@ -231,7 +265,8 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
}
}
validateScope(input.scopeType ?? null, input.scopeValue ?? null);
if (getScriptByName(input.name)) throw new ScriptError(`script "${input.name}" already exists`, 'name-taken');
if (getScriptByName(input.name))
throw new ScriptError(`script "${input.name}" already exists`, 'name-taken');
const sha = sha256(input.content);
const releaseVersion = normalizeReleaseVersion(input.releaseVersion);
@@ -249,9 +284,23 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
let scriptId!: number;
let versionId!: number;
db().transaction(() => {
const s = insertScript.run(input.name, targetRelPath, input.scopeType ?? null, input.scopeValue ?? null, kind);
const s = insertScript.run(
input.name,
targetRelPath,
input.scopeType ?? null,
input.scopeValue ?? null,
kind
);
scriptId = Number(s.lastInsertRowid);
const v = insertVersion.run(scriptId, sha, size, putFileBlob(input.content), input.notes ?? null, kind, releaseVersion);
const v = insertVersion.run(
scriptId,
sha,
size,
putFileBlob(input.content),
input.notes ?? null,
kind,
releaseVersion
);
versionId = Number(v.lastInsertRowid);
})();
@@ -261,7 +310,12 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
};
}
export function uploadNewVersion(scriptId: number, content: Buffer, notes?: string, releaseVersion?: string): ScriptVersionRow {
export function uploadNewVersion(
scriptId: number,
content: Buffer,
notes?: string,
releaseVersion?: string
): ScriptVersionRow {
const script = getScript(scriptId);
if (!script) throw new ScriptError(`no such script (id=${scriptId})`, 'no-such-script');
@@ -285,7 +339,16 @@ export function uploadNewVersion(scriptId: number, content: Buffer, notes?: stri
`INSERT INTO script_versions (script_id, version, sha256, size, content_hash, notes, kind, release_version)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(scriptId, nextVersion, sha, content.byteLength, putFileBlob(content), notes ?? null, script.kind, release);
.run(
scriptId,
nextVersion,
sha,
content.byteLength,
putFileBlob(content),
notes ?? null,
script.kind,
release
);
return getVersion(Number(res.lastInsertRowid))!;
}
@@ -293,7 +356,10 @@ function validateScope(scopeType: ScopeType | null, scopeValue: string | null):
if (scopeType === null && scopeValue === null) return;
if (scopeType === 'game' || scopeType === 'map') {
if (!scopeValue || !scopeValue.trim()) {
throw new ScriptError(`scope_value required for scope_type=${scopeType}`, 'invalid-scope');
throw new ScriptError(
`scope_value required for scope_type=${scopeType}`,
'invalid-scope'
);
}
return;
}
@@ -352,7 +418,11 @@ export function deleteVersion(versionId: number): void {
WHERE bs.script_version_id = ? AND b.state IN ('running','ready-promote') LIMIT 1`
)
.get(versionId) as { id: number } | undefined;
if (beta) throw new ScriptError(`beta ${beta.id} is still running against this version`, 'in-use-beta');
if (beta)
throw new ScriptError(
`beta ${beta.id} is still running against this version`,
'in-use-beta'
);
db().prepare('DELETE FROM script_versions WHERE id = ?').run(versionId);
}
@@ -370,7 +440,11 @@ export function deleteScript(scriptId: number): void {
WHERE bs.script_id = ? AND b.state IN ('running','ready-promote') LIMIT 1`
)
.get(scriptId) as { id: number } | undefined;
if (beta) throw new ScriptError(`beta ${beta.id} is still running against this script`, 'has-active-beta');
if (beta)
throw new ScriptError(
`beta ${beta.id} is still running against this script`,
'has-active-beta'
);
// Every FK that references scripts(id) or script_versions(id) with NO
// ACTION (the default) will abort the delete unless we null it first.
@@ -386,14 +460,37 @@ export function deleteScript(scriptId: number): void {
const handle = db();
const tx = handle.transaction((id: number) => {
const inMyVersions = 'IN (SELECT id FROM script_versions WHERE script_id = ?)';
const inMyBetas = 'IN (SELECT id FROM betas WHERE script_id = ? UNION SELECT beta_id FROM beta_scripts WHERE script_id = ?)';
const inMyBetas =
'IN (SELECT id FROM betas WHERE script_id = ? UNION SELECT beta_id FROM beta_scripts WHERE script_id = ?)';
handle.prepare(`UPDATE beta_servers SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE beta_server_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE deployment_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE deployments SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE deployments SET script_version_id = NULL WHERE script_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE deployments SET beta_id = NULL WHERE beta_id ${inMyBetas}`).run(id, id);
handle
.prepare(
`UPDATE beta_servers SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(
`UPDATE beta_server_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(
`UPDATE deployment_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(
`UPDATE deployments SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(
`UPDATE deployments SET script_version_id = NULL WHERE script_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(`UPDATE deployments SET beta_id = NULL WHERE beta_id ${inMyBetas}`)
.run(id, id);
handle.prepare('UPDATE deployments SET script_id = NULL WHERE script_id = ?').run(id);
handle.prepare('DELETE FROM scripts WHERE id = ?').run(id);
});
@@ -402,7 +499,10 @@ export function deleteScript(scriptId: number): void {
// Servers that match a script's scope. Empty for unscoped scripts — those are
// only ever pushed to explicitly chosen beta targets.
export function serversMatchingScope(scopeType: ScopeType | null, scopeValue: string | null): string[] {
export function serversMatchingScope(
scopeType: ScopeType | null,
scopeValue: string | null
): string[] {
if (scopeType === null || scopeValue === null) return [];
const rows = db()
.prepare(`SELECT name FROM servers WHERE ${scopeType} = ? ORDER BY name`)
@@ -415,7 +515,9 @@ export function serversMatchingScope(scopeType: ScopeType | null, scopeValue: st
// server.
export function listDistinctGames(): string[] {
const rows = db()
.prepare(`SELECT DISTINCT game FROM servers WHERE game IS NOT NULL AND game <> '' ORDER BY game`)
.prepare(
`SELECT DISTINCT game FROM servers WHERE game IS NOT NULL AND game <> '' ORDER BY game`
)
.all() as Array<{ game: string }>;
return rows.map((r) => r.game);
}
src/lib/server/fleet/server-config.ts
+35 -6
View File
@@ -7,13 +7,26 @@ import { applyRuntimePassword } from './runtime-config.js';
export const MAP_CONFIG_FILE = 'electric_cherry_map.cfg';
/** Loaded after Plutainer's default/main config and before map_rotate. */
export function getServerMapConfig(game: string | null, map: string | null, mapSlug?: string | null): Buffer | null {
export function getServerMapConfig(
game: string | null,
map: string | null,
mapSlug?: string | null
): Buffer | null {
const entry = selectedZombieMap(game, map, mapSlug);
return entry ? Buffer.from(`// Generated by Electric Cherry from the selected map.\nset sv_maprotation "${mapRotation(entry)}"\n`) : null;
return entry
? Buffer.from(
`// Generated by Electric Cherry from the selected map.\nset sv_maprotation "${mapRotation(entry)}"\n`
)
: null;
}
/** The promoted config used for first boot and drift reconciliation. */
export function getPromotedServerConfig(game: string | null, map: string | null, mapSlug?: string | null, rconPassword?: string | null): Buffer | null {
export function getPromotedServerConfig(
game: string | null,
map: string | null,
mapSlug?: string | null,
rconPassword?: string | null
): Buffer | null {
if (!game) return null;
const baseline = getBaselineForScope('game', game);
if (!baseline?.current_version_id) return null;
@@ -24,8 +37,24 @@ export function getPromotedServerConfig(game: string | null, map: string | null,
...listPromotedOverlaysForScope('game', game),
...(map ? listPromotedOverlaysForScope('map', map) : [])
].flatMap((overlay) => {
const current = overlay.current_version_id ? getVersion(overlay.current_version_id) : undefined;
return current ? [{ id: overlay.id, text: getVersionContent(current.id).toString('utf8') }] : [];
const current = overlay.current_version_id
? getVersion(overlay.current_version_id)
: undefined;
return current
? [{ id: overlay.id, text: getVersionContent(current.id).toString('utf8') }]
: [];
});
return Buffer.from(applyRuntimePassword(composeServerCfg(getVersionContent(version.id).toString('utf8'), overlays, game, map, mapSlug), rconPassword), 'utf8');
return Buffer.from(
applyRuntimePassword(
composeServerCfg(
getVersionContent(version.id).toString('utf8'),
overlays,
game,
map,
mapSlug
),
rconPassword
),
'utf8'
);
}
src/lib/server/fleet/server-keys.ts
+118 -46
View File
@@ -15,7 +15,12 @@ function validGame(game: unknown): asserts game is KeyGame {
}
function normalizedKey(key: unknown): string {
if (typeof key !== 'string' || !key.trim() || key.trim().length > 512 || /\s/.test(key.trim())) {
if (
typeof key !== 'string' ||
!key.trim() ||
key.trim().length > 512 ||
/\s/.test(key.trim())
) {
throw new Error('Enter a server key without whitespace (maximum 512 characters)');
}
return key.trim();
@@ -29,22 +34,34 @@ function normalizedLabel(label: unknown): string {
}
export function listServerKeys(): ServerKey[] {
const rows = db().prepare('SELECT id,game,label,updated_at FROM server_keys ORDER BY game,label,id').all() as Omit<ServerKey, 'assigned_servers'>[];
const assignments = db().prepare(`SELECT DISTINCT k.id AS key_id,l.server_id FROM server_keys k
const rows = db()
.prepare('SELECT id,game,label,updated_at FROM server_keys ORDER BY game,label,id')
.all() as Omit<ServerKey, 'assigned_servers'>[];
const assignments = db()
.prepare(
`SELECT DISTINCT k.id AS key_id,l.server_id FROM server_keys k
JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key JOIN server_key_leases l ON l.key_id=occupied.id
ORDER BY l.server_id`).all() as { key_id: string; server_id: string }[];
ORDER BY l.server_id`
)
.all() as { key_id: string; server_id: string }[];
const byKey = new Map<string, string[]>();
for (const lease of assignments) {
const servers = byKey.get(lease.key_id) ?? [];
servers.push(lease.server_id);
byKey.set(lease.key_id, servers);
}
return rows.map(row => ({ ...row, assigned_servers: byKey.get(row.id) ?? [] }));
return rows.map((row) => ({ ...row, assigned_servers: byKey.get(row.id) ?? [] }));
}
export function getServerKey(serverId: string): string | undefined {
return (db().prepare(`SELECT k.pluto_key FROM server_key_leases l
JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`).get(serverId) as { pluto_key: string } | undefined)?.pluto_key;
return (
db()
.prepare(
`SELECT k.pluto_key FROM server_key_leases l
JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`
)
.get(serverId) as { pluto_key: string } | undefined
)?.pluto_key;
}
/** Saving a credential twice cannot increase available capacity. */
@@ -52,16 +69,27 @@ export function saveServerKey(game: unknown, key: unknown, label?: unknown): str
validGame(game);
const credential = normalizedKey(key);
const keyLabel = label === undefined ? undefined : normalizedLabel(label);
return db().transaction(() => {
const existing = db().prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?').get(game, credential) as { id: string } | undefined;
if (existing) {
if (keyLabel !== undefined) db().prepare("UPDATE server_keys SET label=?,updated_at=datetime('now') WHERE id=?").run(keyLabel, existing.id);
return existing.id;
}
const id = randomUUID();
db().prepare('INSERT INTO server_keys(id,game,label,pluto_key) VALUES(?,?,?,?)').run(id, game, keyLabel ?? '', credential);
return id;
}).immediate();
return db()
.transaction(() => {
const existing = db()
.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?')
.get(game, credential) as { id: string } | undefined;
if (existing) {
if (keyLabel !== undefined)
db()
.prepare(
"UPDATE server_keys SET label=?,updated_at=datetime('now') WHERE id=?"
)
.run(keyLabel, existing.id);
return existing.id;
}
const id = randomUUID();
db()
.prepare('INSERT INTO server_keys(id,game,label,pluto_key) VALUES(?,?,?,?)')
.run(id, game, keyLabel ?? '', credential);
return id;
})
.immediate();
}
// Compare the credential as well as its row: a mistakenly imported copy in
@@ -70,11 +98,18 @@ const availableKey = `NOT EXISTS (SELECT 1 FROM server_key_leases l
JOIN server_keys occupied ON occupied.id=l.key_id
WHERE occupied.pluto_key=k.pluto_key AND (l.server_id!=@server OR l.is_current=0))`;
export function serverKeyPool(game: KeyGame, serverId?: string): { total: number; available: number } {
export function serverKeyPool(
game: KeyGame,
serverId?: string
): { total: number; available: number } {
validGame(game);
return db().prepare(`SELECT COUNT(*) AS total,
return db()
.prepare(
`SELECT COUNT(*) AS total,
COALESCE(SUM(CASE WHEN ${availableKey} THEN 1 ELSE 0 END),0) AS available
FROM server_keys k WHERE k.game=@game`).get({ game, server: serverId ?? '' }) as { total: number; available: number };
FROM server_keys k WHERE k.game=@game`
)
.get({ game, server: serverId ?? '' }) as { total: number; available: number };
}
export function availableServerKeyCount(game: KeyGame): number {
@@ -85,38 +120,75 @@ export function availableServerKeyCount(game: KeyGame): number {
export function reserveServerKey(serverId: string, game: KeyGame, suppliedKey?: unknown): string {
validGame(game);
if (!serverId) throw new Error('Choose a server for the key reservation');
const credential = suppliedKey === undefined || suppliedKey === null || (typeof suppliedKey === 'string' && !suppliedKey.trim())
? undefined : normalizedKey(suppliedKey);
return db().transaction(() => {
const existing = db().prepare(`SELECT k.game,k.pluto_key FROM server_key_leases l
JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`).get(serverId) as { game: KeyGame; pluto_key: string } | undefined;
if (existing) {
if (existing.game !== game) throw new Error('This server already has a key reserved for another game');
if (credential && credential !== existing.pluto_key) throw new Error('This server already has a reserved key; destroy it before changing its key');
return existing.pluto_key;
}
const requestedId = credential ? saveServerKey(game, credential) : null;
const key = db().prepare(`SELECT k.id,k.pluto_key FROM server_keys k WHERE k.game=@game
const credential =
suppliedKey === undefined ||
suppliedKey === null ||
(typeof suppliedKey === 'string' && !suppliedKey.trim())
? undefined
: normalizedKey(suppliedKey);
return db()
.transaction(() => {
const existing = db()
.prepare(
`SELECT k.game,k.pluto_key FROM server_key_leases l
JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`
)
.get(serverId) as { game: KeyGame; pluto_key: string } | undefined;
if (existing) {
if (existing.game !== game)
throw new Error('This server already has a key reserved for another game');
if (credential && credential !== existing.pluto_key)
throw new Error(
'This server already has a reserved key; destroy it before changing its key'
);
return existing.pluto_key;
}
const requestedId = credential ? saveServerKey(game, credential) : null;
const key = db()
.prepare(
`SELECT k.id,k.pluto_key FROM server_keys k WHERE k.game=@game
AND (@requestedId IS NULL OR k.id=@requestedId) AND ${availableKey}
ORDER BY k.updated_at,k.id LIMIT 1`).get({ game, server: serverId, requestedId }) as { id: string; pluto_key: string } | undefined;
if (!key) throw new Error(credential ? 'This server key is already reserved by another server'
: `No available ${game.toUpperCase()} server keys. Add a key or destroy a server to release one.`);
db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(serverId, key.id);
return key.pluto_key;
}).immediate();
ORDER BY k.updated_at,k.id LIMIT 1`
)
.get({ game, server: serverId, requestedId }) as
{ id: string; pluto_key: string } | undefined;
if (!key)
throw new Error(
credential
? 'This server key is already reserved by another server'
: `No available ${game.toUpperCase()} server keys. Add a key or destroy a server to release one.`
);
db()
.prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)')
.run(serverId, key.id);
return key.pluto_key;
})
.immediate();
}
/** Only after confirmed destruction or a creation cancelled before any host mutation. */
export function releaseServerKey(serverId: string): boolean {
return db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(serverId).changes > 0;
return (
db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(serverId).changes > 0
);
}
export function removeServerKey(id: string): boolean {
return db().transaction(() => {
if (db().prepare(`SELECT 1 FROM server_keys k JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key
JOIN server_key_leases l ON l.key_id=occupied.id WHERE k.id=?`).get(id)) {
throw new Error('This key is reserved by a server. Destroy that server before removing its key.');
}
return db().prepare('DELETE FROM server_keys WHERE id=?').run(id).changes > 0;
}).immediate();
return db()
.transaction(() => {
if (
db()
.prepare(
`SELECT 1 FROM server_keys k JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key
JOIN server_key_leases l ON l.key_id=occupied.id WHERE k.id=?`
)
.get(id)
) {
throw new Error(
'This key is reserved by a server. Destroy that server before removing its key.'
);
}
return db().prepare('DELETE FROM server_keys WHERE id=?').run(id).changes > 0;
})
.immediate();
}
src/lib/server/fleet/servers.ts
+56 -16
View File
@@ -35,8 +35,23 @@ export function matchServers(all: string[], patterns: string[]): string[] {
return [...all];
}
const regexes = patterns
.flatMap((p) => p.split(',').map((s) => s.trim()).filter(Boolean))
.map((p) => new RegExp('^' + p.replace(/[.+^$()|{}\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.') + '$'));
.flatMap((p) =>
p
.split(',')
.map((s) => s.trim())
.filter(Boolean)
)
.map(
(p) =>
new RegExp(
'^' +
p
.replace(/[.+^$()|{}\\]/g, '\\$&')
.replace(/\*/g, '.*')
.replace(/\?/g, '.') +
'$'
)
);
return all.filter((name) => regexes.some((r) => r.test(name)));
}
@@ -49,8 +64,13 @@ export async function ensureDest(
): Promise<{ ok: true; path: string } | { ok: false; reason: string }> {
const dest = destOf(server);
if (nativeInstance(server)) {
try { return (await managedStat(dest)).isDirectory() ? { ok: true, path: dest } : { ok: false, reason: 'Agent script directory is unavailable' }; }
catch (error) { return { ok: false, reason: (error as Error).message }; }
try {
return (await managedStat(dest)).isDirectory()
? { ok: true, path: dest }
: { ok: false, reason: 'Agent script directory is unavailable' };
} catch (error) {
return { ok: false, reason: (error as Error).message };
}
}
if (existsSync(dest)) {
try {
@@ -152,22 +172,34 @@ export async function seedServerRegistry(): Promise<number> {
}
export function getServer(name: string): ServerRow | undefined {
return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.name=?`).get(name) as ServerRow | undefined;
return db()
.prepare(
`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.name=?`
)
.get(name) as ServerRow | undefined;
}
export function listServers(): ServerRow[] {
return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.destroyed_at IS NULL ORDER BY s.name`).all() as ServerRow[];
return db()
.prepare(
`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.destroyed_at IS NULL ORDER BY s.name`
)
.all() as ServerRow[];
}
/** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */
export function activeBetaRemovalBlocker(server: string): string | null {
const betas = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
const betas = db()
.prepare(
`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')
ORDER BY b.id`).all(server) as { id: number }[];
ORDER BY b.id`
)
.all(server) as { id: number }[];
if (!betas.length) return null;
return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map(beta => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`;
return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map((beta) => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`;
}
export function setServerField(
@@ -178,7 +210,10 @@ export function setServerField(
setServerFields(name, { [field]: value });
}
export function setServerFields(name: string, fields: Partial<Record<keyof ServerRow, string | number | null>>): void {
export function setServerFields(
name: string,
fields: Partial<Record<keyof ServerRow, string | number | null>>
): void {
const allowed: Array<keyof ServerRow> = [
'container_state',
'container_checked_at',
@@ -211,10 +246,15 @@ export function setServerFields(name: string, fields: Partial<Record<keyof Serve
];
const entries = Object.entries(fields);
if (!entries.length) return;
for (const [field] of entries) if (!allowed.includes(field as keyof ServerRow)) throw new Error(`refusing to set ${field}`);
db().prepare(`UPDATE servers SET ${entries.map(([field]) => `${field} = ?`).join(', ')} WHERE name = ?
AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})`)
.run(...entries.map(([,value]) => value), name, ...entries.map(([,value]) => value));
for (const [field] of entries)
if (!allowed.includes(field as keyof ServerRow))
throw new Error(`refusing to set ${field}`);
db()
.prepare(
`UPDATE servers SET ${entries.map(([field]) => `${field} = ?`).join(', ')} WHERE name = ?
AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})`
)
.run(...entries.map(([, value]) => value), name, ...entries.map(([, value]) => value));
}
/**
src/lib/server/fleet/snapshot.ts
+55 -14
View File
@@ -15,32 +15,62 @@ export function fleetSnapshot(serverName?: string) {
const hosts = listHosts();
const serverNow = Date.now();
const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host]));
const versions = new Map(runningVersions(serverName).map(version => [version.server, version]));
const instances = new Map((db().prepare(`SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
.all(...bindings) as NativeInstance[]).map(instance => [instance.server, instance]));
const versions = new Map(
runningVersions(serverName).map((version) => [version.server, version])
);
const instances = new Map(
(
db()
.prepare(
`SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`
)
.all(...bindings) as NativeInstance[]
).map((instance) => [instance.server, instance])
);
const holds = new Map<string, { target: string; betaId: number }[]>();
for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
for (const row of db()
.prepare(
`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id
JOIN beta_server_scripts bs ON bs.beta_id=b.id AND bs.server=bsv.server
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')
${serverName === undefined ? '' : 'AND bsv.server = ?'}`).all(...bindings) as
{ server: string; target: string; betaId: number }[]) {
const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries);
${serverName === undefined ? '' : 'AND bsv.server = ?'}`
)
.all(...bindings) as { server: string; target: string; betaId: number }[]) {
const entries = holds.get(row.server) ?? [];
entries.push({ target: row.target, betaId: row.betaId });
holds.set(row.server, entries);
}
const observe = serverName === undefined ? observationReader() : serverObservation;
const capacity = playerCapacityReader(serverName);
const selected = serverName === undefined ? undefined : getServer(serverName);
const rows = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
const rows =
serverName === undefined
? listServers()
: selected && !selected.destroyed_at
? [selected]
: [];
const servers = rows.map((s) => {
const instance = s.native_instance_id ? instances.get(s.name) : null;
return {
...observe(s),
...capacity(s.name),
host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
host: s.host_id ? (hostsById.get(s.host_id) ?? null) : null,
network: instance
? {
bindAddress: instance.bind_address,
localPort: instance.port,
publicAddress: instance.public_address,
publicPort: instance.public_port
}
: null,
plutonium: versions.get(s.name),
// The host agent owns the game connection.
endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
endpoint: instance
? { host: `agent:${instance.host_id}`, port: instance.port }
: s.game_port
? { host: s.rcon_host ?? rconHost, port: s.game_port }
: null,
held: holds.get(s.name) ?? []
};
});
@@ -49,11 +79,22 @@ export function fleetSnapshot(serverName?: string) {
serverNow,
// Include hosts without assigned servers. Re-evaluate heartbeat age on
// every existing snapshot refresh rather than adding a navigation poll.
hostAttention: hosts.filter(host => ['attention', 'offline'].includes(hostListStatus(host, serverNow).key)).length,
reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [],
hostAttention: hosts.filter((host) =>
['attention', 'offline'].includes(hostListStatus(host, serverNow).key)
).length,
reservedNames:
serverName === undefined
? (
db()
.prepare(
"SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')"
)
.all() as { server: string }[]
).map((row) => row.server)
: [],
integrations: {
native: true,
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
hostsReady: hosts.filter((host) => host.readiness === 'ready').length,
hostsTotal: hosts.length,
rcon: rconConfigured(),
rconTimeoutMs: config.rcon.timeoutMs,
src/lib/server/fleet/staged-files.ts
+199 -59
View File
@@ -21,26 +21,38 @@ interface FileChange {
expectedBeforeSha?: string | null;
}
export interface StagedFileChange {
id: number; source_kind: SourceKind; source_id: string; server: string;
kind: ArtifactKind; rel_path: string; content: Buffer | null;
script_version_id: number | null; inventory_source: string;
deployment_file_id: number | null; preserve_before_image: number;
check_before: number; expected_before_sha: string | null;
state: 'pending' | 'applying' | 'applied' | 'cancelled'; applied_at: string | null;
id: number;
source_kind: SourceKind;
source_id: string;
server: string;
kind: ArtifactKind;
rel_path: string;
content: Buffer | null;
script_version_id: number | null;
inventory_source: string;
deployment_file_id: number | null;
preserve_before_image: number;
check_before: number;
expected_before_sha: string | null;
state: 'pending' | 'applying' | 'applied' | 'cancelled';
applied_at: string | null;
}
function targetPath(server: string, kind: ArtifactKind, relPath: string): string {
const root = resolve(destOfKind(server, kind));
const target = resolve(root, relPath);
const rel = relative(root, target);
if (!rel || rel === '..' || rel.startsWith('../') || isAbsolute(rel)) throw new Error('File target must be inside its artifact directory');
if (!rel || rel === '..' || rel.startsWith('../') || isAbsolute(rel))
throw new Error('File target must be inside its artifact directory');
return target;
}
/** Capture immutable desired bytes without changing the server or its inventory. */
export function queueFileChange(change: FileChange): void {
targetPath(change.server, change.kind, change.relPath);
db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content_hash,
db()
.prepare(
`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content_hash,
script_version_id, inventory_source, deployment_file_id, preserve_before_image, check_before, expected_before_sha)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(source_kind, source_id, server, kind, rel_path) DO UPDATE SET
@@ -48,51 +60,122 @@ export function queueFileChange(change: FileChange): void {
inventory_source = excluded.inventory_source, deployment_file_id = excluded.deployment_file_id,
preserve_before_image = excluded.preserve_before_image, check_before = excluded.check_before,
expected_before_sha = excluded.expected_before_sha
WHERE staged_file_changes.state = 'pending'`)
.run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, putFileBlob(change.content),
change.scriptVersionId ?? null, change.inventorySource, change.deploymentFileId ?? null,
change.preserveBeforeImage ? 1 : 0, change.expectedBeforeSha !== undefined ? 1 : 0, change.expectedBeforeSha ?? null);
WHERE staged_file_changes.state = 'pending'`
)
.run(
change.sourceKind,
change.sourceId,
change.server,
change.kind,
change.relPath,
putFileBlob(change.content),
change.scriptVersionId ?? null,
change.inventorySource,
change.deploymentFileId ?? null,
change.preserveBeforeImage ? 1 : 0,
change.expectedBeforeSha !== undefined ? 1 : 0,
change.expectedBeforeSha ?? null
);
}
export function stagedFileChanges(sourceKind: SourceKind, sourceId: string, server?: string): StagedFileChange[] {
return db().prepare(`SELECT *, ${blobContent('content_hash')} AS content FROM staged_file_changes WHERE source_kind = ? AND source_id = ?
${server === undefined ? '' : 'AND server = ?'} ORDER BY id`)
export function stagedFileChanges(
sourceKind: SourceKind,
sourceId: string,
server?: string
): StagedFileChange[] {
return db()
.prepare(
`SELECT *, ${blobContent('content_hash')} AS content FROM staged_file_changes WHERE source_kind = ? AND source_id = ?
${server === undefined ? '' : 'AND server = ?'} ORDER BY id`
)
.all(sourceKind, sourceId, ...(server === undefined ? [] : [server])) as StagedFileChange[];
}
export function cancelStagedFiles(sourceKind: SourceKind, sourceId: string, server?: string): void {
db().prepare(`UPDATE staged_file_changes SET state = 'cancelled' WHERE source_kind = ? AND source_id = ?
AND state = 'pending' ${server === undefined ? '' : 'AND server = ?'}`)
db()
.prepare(
`UPDATE staged_file_changes SET state = 'cancelled' WHERE source_kind = ? AND source_id = ?
AND state = 'pending' ${server === undefined ? '' : 'AND server = ?'}`
)
.run(sourceKind, sourceId, ...(server === undefined ? [] : [server]));
}
export function hasAppliedFiles(sourceKind: SourceKind, sourceId: string, server?: string): boolean {
return stagedFileChanges(sourceKind, sourceId, server).some(row => row.state === 'applied' || row.state === 'applying');
export function hasAppliedFiles(
sourceKind: SourceKind,
sourceId: string,
server?: string
): boolean {
return stagedFileChanges(sourceKind, sourceId, server).some(
(row) => row.state === 'applied' || row.state === 'applying'
);
}
/** null identifies legacy work that wrote files before staging was introduced. */
export function isFileApplied(sourceKind: SourceKind, sourceId: string, server: string, kind: ArtifactKind, relPath: string): boolean | null {
const row = stagedFileChanges(sourceKind, sourceId, server).find(row => row.kind === kind && row.rel_path === relPath);
export function isFileApplied(
sourceKind: SourceKind,
sourceId: string,
server: string,
kind: ArtifactKind,
relPath: string
): boolean | null {
const row = stagedFileChanges(sourceKind, sourceId, server).find(
(row) => row.kind === kind && row.rel_path === relPath
);
return row ? row.state === 'applied' || row.state === 'applying' : null;
}
function readExisting(path: string): Buffer | null {
try { return readFileSync(path); }
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err; }
try {
return readFileSync(path);
} catch (err) {
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw err;
}
}
/** The first activation owns the rollback image, including an absent file. */
export function captureBetaBeforeImage(row: Pick<StagedFileChange, 'source_id' | 'server' | 'rel_path' | 'kind'>, before: Buffer | null): void {
if (db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?')
.get(row.source_id, row.server, row.rel_path)) return;
export function captureBetaBeforeImage(
row: Pick<StagedFileChange, 'source_id' | 'server' | 'rel_path' | 'kind'>,
before: Buffer | null
): void {
if (
db()
.prepare(
'SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?'
)
.get(row.source_id, row.server, row.rel_path)
)
return;
const prior = getServerScript(row.server, row.rel_path);
if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots
(beta_id, server, rel_path, prior_content_hash, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
.run(row.source_id, row.server, row.rel_path, putFileBlob(before), sha256Buffer(before), before.length);
db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`)
.run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, row.server, row.rel_path, row.kind);
db().prepare('INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
if (before !== null)
db()
.prepare(
`INSERT OR IGNORE INTO beta_file_snapshots
(beta_id, server, rel_path, prior_content_hash, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`
)
.run(
row.source_id,
row.server,
row.rel_path,
putFileBlob(before),
sha256Buffer(before),
before.length
);
db()
.prepare(
`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`
)
.run(
before === null ? 0 : 1,
prior?.script_version_id ?? null,
row.source_id,
row.server,
row.rel_path,
row.kind
);
db()
.prepare('INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
.run(row.source_id, row.server, row.rel_path);
}
@@ -101,25 +184,44 @@ export function captureBetaBeforeImage(row: Pick<StagedFileChange, 'source_id' |
* from interleaving between the final gates and dispatching the restart.
*/
export function applyStagedFilesForOperation(operationId: string, server: string): void {
const rows = db().prepare(`SELECT f.*, ${blobContent('f.content_hash')} AS content FROM staged_file_changes f JOIN provisioning_sources s
const rows = db()
.prepare(
`SELECT f.*, ${blobContent('f.content_hash')} AS content FROM staged_file_changes f JOIN provisioning_sources s
ON s.kind = f.source_kind AND s.source_id = f.source_id AND s.server = f.server
WHERE s.operation_id = ? AND f.server = ? AND f.state IN ('pending','applying') ORDER BY f.id`)
WHERE s.operation_id = ? AND f.server = ? AND f.state IN ('pending','applying') ORDER BY f.id`
)
.all(operationId, server) as StagedFileChange[];
if (!rows.length) return;
if (!statSync(serverDir(server)).isDirectory()) throw new Error('Server directory is unavailable');
if (!statSync(serverDir(server)).isDirectory())
throw new Error('Server directory is unavailable');
// Preflight every target before changing any files. An interrupted write
// needs inspection/rollback, never an automatic replay.
const plans = rows.map(row => {
if (row.state === 'applying') throw new Error('A previous file write was interrupted; inspect or roll back the campaign');
const source = row.source_kind === 'campaign'
? db().prepare(`SELECT 1 FROM deployment_servers WHERE deployment_id = ? AND server = ?
AND server_deleted_at IS NULL AND provisioning_id = ? AND restart_state IN ('pending','failed')`).get(row.source_id, server, operationId)
: db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.beta_id = ? AND bs.server = ?
AND bs.server_deleted_at IS NULL AND bs.provisioning_id = ? AND b.state IN ('running','ready-promote')`).get(row.source_id, server, operationId);
const plans = rows.map((row) => {
if (row.state === 'applying')
throw new Error(
'A previous file write was interrupted; inspect or roll back the campaign'
);
const source =
row.source_kind === 'campaign'
? db()
.prepare(
`SELECT 1 FROM deployment_servers WHERE deployment_id = ? AND server = ?
AND server_deleted_at IS NULL AND provisioning_id = ? AND restart_state IN ('pending','failed')`
)
.get(row.source_id, server, operationId)
: db()
.prepare(
`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.beta_id = ? AND bs.server = ?
AND bs.server_deleted_at IS NULL AND bs.provisioning_id = ? AND b.state IN ('running','ready-promote')`
)
.get(row.source_id, server, operationId);
if (!source) throw new Error('File change requester is no longer eligible');
const path = targetPath(server, row.kind, row.rel_path);
const before = readExisting(path);
if (row.check_before && (before === null ? null : sha256Buffer(before)) !== row.expected_before_sha) {
if (
row.check_before &&
(before === null ? null : sha256Buffer(before)) !== row.expected_before_sha
) {
throw new Error(`${row.rel_path}: changed while queued; scan and review again`);
}
return { row, path, before };
@@ -132,14 +234,30 @@ export function applyStagedFilesForOperation(operationId: string, server: string
captureBetaBeforeImage(row, before);
}
if (row.deployment_file_id && !row.preserve_before_image) {
db().prepare(`UPDATE deployment_files SET
verb = ?, content_before_hash = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`)
.run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', putFileBlob(before),
before === null ? null : sha256Buffer(before), row.deployment_file_id);
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${server}: captured rollback snapshot for ${row.rel_path}`);
db()
.prepare(
`UPDATE deployment_files SET
verb = ?, content_before_hash = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`
)
.run(
row.content === null ? 'removed' : before === null ? 'added' : 'updated',
putFileBlob(before),
before === null ? null : sha256Buffer(before),
row.deployment_file_id
);
if (row.source_kind === 'campaign')
db()
.prepare(
'INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)'
)
.run(
row.source_id,
`${server}: captured rollback snapshot for ${row.rel_path}`
);
}
db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?").run(row.id);
db()
.prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?")
.run(row.id);
})();
if (row.content === null) {
rmSync(path, { force: true });
@@ -151,17 +269,39 @@ export function applyStagedFilesForOperation(operationId: string, server: string
writeFileSync(path, row.content);
// A queued version can be deleted independently; the staged bytes
// still apply, but its optional inventory link must remain valid.
const version = row.script_version_id !== null && db().prepare('SELECT 1 FROM script_versions WHERE id = ?').get(row.script_version_id)
? row.script_version_id : null;
recordScript(server, row.rel_path, version, sha256Buffer(row.content), row.content.length, row.inventory_source);
const version =
row.script_version_id !== null &&
db()
.prepare('SELECT 1 FROM script_versions WHERE id = ?')
.get(row.script_version_id)
? row.script_version_id
: null;
recordScript(
server,
row.rel_path,
version,
sha256Buffer(row.content),
row.content.length,
row.inventory_source
);
}
db().prepare("UPDATE staged_file_changes SET state = 'applied', applied_at = ? WHERE id = ?")
db()
.prepare(
"UPDATE staged_file_changes SET state = 'applied', applied_at = ? WHERE id = ?"
)
.run(new Date().toISOString(), row.id);
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${server}: applied ${row.rel_path}`);
if (row.source_kind === 'campaign')
db()
.prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${server}: applied ${row.rel_path}`);
}
for (const betaId of new Set(rows.filter(row => row.source_kind === 'beta').map(row => row.source_id))) {
db().prepare('UPDATE beta_servers SET deployed_at = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL')
for (const betaId of new Set(
rows.filter((row) => row.source_kind === 'beta').map((row) => row.source_id)
)) {
db()
.prepare(
'UPDATE beta_servers SET deployed_at = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL'
)
.run(new Date().toISOString(), new Date().toISOString(), betaId, server);
}
}
src/lib/server/fleet/status-evidence.ts
+68 -22
View File
@@ -1,32 +1,49 @@
import { db } from '../db/index.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { rconGetPlayerCount, rconStatusResult, type RconStatusResult } from '../integrations/rcon.js';
import {
rconGetPlayerCount,
rconStatusResult,
type RconStatusResult
} from '../integrations/rcon.js';
import { latestGameSubmission } from './lifecycle-evidence.js';
export { latestGameSubmission } from './lifecycle-evidence.js';
export interface TelemetryEvidence { players: number; map: string; receivedAt: string }
export interface TelemetryEvidence {
players: number;
map: string;
receivedAt: string;
}
export function telemetryReport(server: string): GameTelemetry | null {
const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as
{ received_at: string; payload: string | null } | undefined;
const row = db()
.prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?')
.get(server) as { received_at: string; payload: string | null } | undefined;
if (!row?.payload) return null;
try {
const saved = JSON.parse(row.payload) as GameTelemetry;
const event = parseGameEvent(saved);
return event.server === server ? { ...event, receivedAt: row.received_at, timing: saved.timing } : null;
} catch { return null; }
return event.server === server
? { ...event, receivedAt: row.received_at, timing: saved.timing }
: null;
} catch {
return null;
}
}
/** Only validated, recently received reports can establish current game state. */
export function telemetryEvidence(server: string, after: string | null = latestGameSubmission(server)): TelemetryEvidence | null {
export function telemetryEvidence(
server: string,
after: string | null = latestGameSubmission(server)
): TelemetryEvidence | null {
const report = telemetryReport(server);
if (!report) return null;
const received = Date.parse(report.receivedAt);
const age = Date.now() - received;
if (!Number.isFinite(age) || age < 0 || age > ONLINE_MAX_MS) return null;
if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after))) return null;
if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after)))
return null;
return { players: report.players.length, map: report.map, receivedAt: report.receivedAt };
}
@@ -45,39 +62,68 @@ export interface PlayerEvidence {
/** The same eligibility decision, with provenance from the probes already performed. */
export async function probePlayerEvidence(server: string): Promise<PlayerEvidence> {
const fromTelemetry = (value: TelemetryEvidence): PlayerEvidence => ({
players: value.players, source: 'telemetry', observedAt: new Date().toISOString(), receivedAt: value.receivedAt
players: value.players,
source: 'telemetry',
observedAt: new Date().toISOString(),
receivedAt: value.receivedAt
});
const occupied = telemetryEvidence(server);
if (occupied && occupied.players > 0) return fromTelemetry(occupied);
const players = await rconGetPlayerCount(server).catch(() => null);
// Read after the network request: a player may join while RCON is pending.
const telemetry = telemetryEvidence(server);
if (players !== null && players > 0) return { players, source: 'rcon', observedAt: new Date().toISOString() };
if (players !== null && players > 0)
return { players, source: 'rcon', observedAt: new Date().toISOString() };
if (telemetry && telemetry.players > 0) return fromTelemetry(telemetry);
if (players === 0) return { players, source: 'rcon', observedAt: new Date().toISOString() };
if (telemetry?.players === 0) return fromTelemetry(telemetry);
return { players: null, source: null, observedAt: new Date().toISOString() };
}
export type VerifiedGameStatus = { ok: true; players: number; map: string | null; observedAt: string } & (
{ source: 'rcon'; rcon: Extract<RconStatusResult, { ok: true }> }
| { source: 'telemetry'; receivedAt: string });
export type VerifiedGameStatus = {
ok: true;
players: number;
map: string | null;
observedAt: string;
} & (
| { source: 'rcon'; rcon: Extract<RconStatusResult, { ok: true }> }
| { source: 'telemetry'; receivedAt: string }
);
/** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */
export async function probeServerStatus(server: string, after?: string | null): Promise<
VerifiedGameStatus | { ok: false; reason: string; observedAt: string }
> {
export async function probeServerStatus(
server: string,
after?: string | null
): Promise<VerifiedGameStatus | { ok: false; reason: string; observedAt: string }> {
const current = telemetryEvidence(server, after);
if (current) return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() };
let reason = 'RCON unavailable';
if (current)
return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() };
let reason: string;
try {
const rcon = await rconStatusResult(server);
const latest = telemetryEvidence(server, after);
if (latest) return { ok: true, ...latest, source: 'telemetry', observedAt: new Date().toISOString() };
if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', rcon, observedAt: new Date().toISOString() };
if (latest)
return {
ok: true,
...latest,
source: 'telemetry',
observedAt: new Date().toISOString()
};
if (rcon.ok)
return {
ok: true,
players: rcon.status.players,
map: rcon.status.map,
source: 'rcon',
rcon,
observedAt: new Date().toISOString()
};
reason = rcon.reason;
} catch (err) { reason = (err as Error).message; }
} catch (err) {
reason = (err as Error).message;
}
const telemetry = telemetryEvidence(server, after);
return telemetry ? { ok: true, ...telemetry, source: 'telemetry', observedAt: new Date().toISOString() }
return telemetry
? { ok: true, ...telemetry, source: 'telemetry', observedAt: new Date().toISOString() }
: { ok: false, reason, observedAt: new Date().toISOString() };
}
src/lib/server/fleet/status-updates.ts
+19 -7
View File
@@ -3,12 +3,20 @@ import type { ServerObservation } from '../../server-observation.js';
import type { ServerRow } from './servers.js';
/** Monitoring and provisioning publish the same complete RCON observation. */
export function successfulStatusFields(result: Extract<RconStatusResult, { ok: true }>, observedAt: string): Partial<ServerRow> {
export function successfulStatusFields(
result: Extract<RconStatusResult, { ok: true }>,
observedAt: string
): Partial<ServerRow> {
return {
last_checked_at: observedAt, last_seen_at: observedAt,
last_players: result.status.players, last_map: result.status.map,
last_gametype: result.status.gametype, last_roster: JSON.stringify(result.status.roster),
last_reply_snippet: null, last_error: null, rcon_failures: 0
last_checked_at: observedAt,
last_seen_at: observedAt,
last_players: result.status.players,
last_map: result.status.map,
last_gametype: result.status.gametype,
last_roster: JSON.stringify(result.status.roster),
last_reply_snippet: null,
last_error: null,
rcon_failures: 0
};
}
@@ -17,6 +25,10 @@ export function activityFields(current: ServerRow, status: ServerObservation): P
if (status.health !== 'online' || status.last_players === null) return {};
return status.last_players > 0
? { active_since: current.active_since ?? status.last_seen_at, inactive_since: null }
: { active_since: null, inactive_since: current.inactive_since ?? status.last_seen_at,
game_started_at: null, game_ended_at: null };
: {
active_since: null,
inactive_since: current.inactive_since ?? status.last_seen_at,
game_started_at: null,
game_ended_at: null
};
}
src/lib/server/fleet/status.ts
+96 -28
View File
@@ -1,4 +1,8 @@
import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js';
import {
rconHealthError,
selectServerObservation,
type GameObservation
} from '../../server-observation.js';
import { classifyHealth } from './health.js';
import { isProvisioning } from './provisioning.js';
import { restartSettling, recoverySettling } from '../provisioning/restarts.js';
@@ -8,32 +12,71 @@ import { latestGameSubmissions } from './lifecycle-evidence.js';
import { db } from '../db/index.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null }
interface Evidence {
provisioning: boolean;
settling: boolean;
telemetry: GameTelemetry | null;
submission: string | null;
}
/** One set of evidence reads per snapshot, rather than four queries per server. */
export function observationReader() {
const handle = db();
const operations = new Map((handle.prepare(`SELECT id, server, action, state FROM server_deployments
WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as
{ id: string; server: string; action: string; state: string }[]).map(row => [row.server, row]));
const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout').all() as
{ server: string; requested_at: string; healthy_at: string | null; operation_id: string | null }[]).map(row => [row.server, row]));
const operations = new Map(
(
handle
.prepare(
`SELECT id, server, action, state FROM server_deployments
WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`
)
.all() as { id: string; server: string; action: string; state: string }[]
).map((row) => [row.server, row])
);
const restarts = new Map(
(
handle
.prepare(
'SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout'
)
.all() as {
server: string;
requested_at: string;
healthy_at: string | null;
operation_id: string | null;
}[]
).map((row) => [row.server, row])
);
const submissions = latestGameSubmissions();
const reports = new Map<string, GameTelemetry>();
for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
{ server: string; received_at: string; payload: string }[]) {
for (const row of handle
.prepare(
'SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL'
)
.all() as { server: string; received_at: string; payload: string }[]) {
try {
const saved = JSON.parse(row.payload) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.server) reports.set(row.server, { ...event, receivedAt: row.received_at, timing: saved.timing });
} catch { /* Invalid retained telemetry is unavailable. */ }
if (event.server === row.server)
reports.set(row.server, {
...event,
receivedAt: row.received_at,
timing: saved.timing
});
} catch {
/* Invalid retained telemetry is unavailable. */
}
}
return (server: ServerRow) => {
const operation = operations.get(server.name), restart = restarts.get(server.name);
const operation = operations.get(server.name),
restart = restarts.get(server.name);
return serverObservation(server, {
provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
provisioning:
!!operation &&
['create', 'rebuild', 'start'].includes(operation.action) &&
['queued', 'running', 'awaiting-health'].includes(operation.state),
settling: recoverySettling(restart, operation),
telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
telemetry: reports.get(server.name) ?? null,
submission: submissions.get(server.name) ?? null
});
};
}
@@ -41,22 +84,47 @@ export function observationReader() {
/** Shared snapshot for REST, status events, metrics, and other server-side readers. */
export function serverObservation(server: ServerRow, evidence?: Evidence) {
let roster = null;
try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error,
rcon_failures: server.rcon_failures, roster };
try {
roster = server.last_roster ? JSON.parse(server.last_roster) : null;
} catch {
/* unavailable */
}
const rcon: GameObservation = {
last_players: server.last_players,
last_map: server.last_map,
last_gametype: server.last_gametype,
last_seen_at: server.last_seen_at,
last_error: server.last_error,
rcon_failures: server.rcon_failures,
roster
};
const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
const state = (evidence ? evidence.settling : restartSettling(server.name))
? 'restarting'
: server.container_state;
const ready = !provisioning && (!state || state === 'running' || state === 'unknown');
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, rconHealthError(rcon), state);
const health = provisioning
? 'provisioning'
: classifyHealth(server.last_seen_at, rconHealthError(rcon), state);
// This projection is shared by viewer-visible snapshots. Never spread saved diagnostics.
const { last_reply_snippet: _snippet, ...publicServer } = server;
return selectServerObservation({ ...publicServer, ...rcon, rcon_status: rcon, container_state: state, health,
status_source: null as 'telemetry' | 'rcon' | null,
telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,
telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name),
last_error: ready ? server.last_error : null,
active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null,
game_started_at: ready ? server.game_started_at : null, game_ended_at: ready ? server.game_ended_at : null
}, undefined, Date.now());
return selectServerObservation(
{
...publicServer,
...rcon,
rcon_status: rcon,
container_state: state,
health,
status_source: null as 'telemetry' | 'rcon' | null,
telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,
telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name),
last_error: ready ? server.last_error : null,
active_since: ready ? server.active_since : null,
inactive_since: ready ? server.inactive_since : null,
game_started_at: ready ? server.game_started_at : null,
game_ended_at: ready ? server.game_ended_at : null
},
undefined,
Date.now()
);
}
src/lib/server/fleet/telemetry-requests.ts
+53 -20
View File
@@ -6,11 +6,18 @@ const MAX_PAYLOAD_LENGTH = 16 * 1024;
// Bound storage even when requests claim many different, unregistered names.
export function telemetryRetention() {
const settings = fleetSettings();
return { maxAgeDays: settings.telemetryKeepDays, maxPerServer: settings.telemetryMaxPerServer, maxTotal: settings.telemetryMaxTotal };
return {
maxAgeDays: settings.telemetryKeepDays,
maxPerServer: settings.telemetryMaxPerServer,
maxTotal: settings.telemetryMaxTotal
};
}
export const telemetryCutoff = () => telemetryRetention().maxAgeDays ? new Date(Date.now() - telemetryRetention().maxAgeDays * 86_400_000).toISOString() : '';
export const telemetryCutoff = () =>
telemetryRetention().maxAgeDays
? new Date(Date.now() - telemetryRetention().maxAgeDays * 86_400_000).toISOString()
: '';
const cutoff = telemetryCutoff;
const textField = (value: unknown) => typeof value === 'string' ? value.slice(0, 255) : null;
const textField = (value: unknown) => (typeof value === 'string' ? value.slice(0, 255) : null);
export function recordTelemetryRequest(input: {
value: unknown;
@@ -27,29 +34,55 @@ export function recordTelemetryRequest(input: {
const payload = JSON.stringify(body);
const handle = db();
handle.transaction(() => {
handle.prepare(`INSERT INTO telemetry_requests
handle
.prepare(
`INSERT INTO telemetry_requests
(server, received_at, status, error, event, map, round, player_count, content_type, payload, payload_truncated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
body.server, input.receivedAt, input.status, input.error?.slice(0, 2000) ?? null,
textField(body.event), textField(body.map),
Number.isSafeInteger(body.round) ? body.round as number : null,
Array.isArray(body.players) ? body.players.length : null,
textField(input.contentType), payload.slice(0, MAX_PAYLOAD_LENGTH), Number(payload.length > MAX_PAYLOAD_LENGTH)
);
if (telemetryRetention().maxAgeDays) handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff());
if (telemetryRetention().maxPerServer) handle.prepare(`DELETE FROM telemetry_requests WHERE server = ? AND id <= (
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(
body.server,
input.receivedAt,
input.status,
input.error?.slice(0, 2000) ?? null,
textField(body.event),
textField(body.map),
Number.isSafeInteger(body.round) ? (body.round as number) : null,
Array.isArray(body.players) ? body.players.length : null,
textField(input.contentType),
payload.slice(0, MAX_PAYLOAD_LENGTH),
Number(payload.length > MAX_PAYLOAD_LENGTH)
);
if (telemetryRetention().maxAgeDays)
handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff());
if (telemetryRetention().maxPerServer)
handle
.prepare(
`DELETE FROM telemetry_requests WHERE server = ? AND id <= (
SELECT id FROM telemetry_requests WHERE server = ? ORDER BY id DESC LIMIT 1 OFFSET ?
)`).run(body.server, body.server, telemetryRetention().maxPerServer);
if (telemetryRetention().maxTotal) handle.prepare(`DELETE FROM telemetry_requests WHERE id <= (
)`
)
.run(body.server, body.server, telemetryRetention().maxPerServer);
if (telemetryRetention().maxTotal)
handle
.prepare(
`DELETE FROM telemetry_requests WHERE id <= (
SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 1 OFFSET ?
)`).run(telemetryRetention().maxTotal);
)`
)
.run(telemetryRetention().maxTotal);
})();
}
export function listTelemetryRequests(server: string): TelemetryRequestLogEntry[] {
const rows = db().prepare(`SELECT id, server, received_at AS receivedAt, status, error, event, map, round,
const rows = db()
.prepare(
`SELECT id, server, received_at AS receivedAt, status, error, event, map, round,
player_count AS playerCount, content_type AS contentType, payload, payload_truncated AS payloadTruncated
FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`)
.all(server, cutoff(), telemetryRetention().maxPerServer || 10000) as Array<Omit<TelemetryRequestLogEntry, 'payloadTruncated'> & { payloadTruncated: number }>;
return rows.map(row => ({ ...row, payloadTruncated: !!row.payloadTruncated }));
FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`
)
.all(server, cutoff(), telemetryRetention().maxPerServer || 10000) as Array<
Omit<TelemetryRequestLogEntry, 'payloadTruncated'> & { payloadTruncated: number }
>;
return rows.map((row) => ({ ...row, payloadTruncated: !!row.payloadTruncated }));
}
src/lib/server/fleet/telemetry.ts
+89 -28
View File
@@ -5,58 +5,119 @@ import { recordLeaderboardTelemetry } from './leaderboard-ingest.js';
import { latestGameSubmission } from './status-evidence.js';
export function listTelemetry(): GameTelemetry[] {
const rows = db().prepare(`SELECT t.payload FROM server_telemetry t JOIN servers s ON s.name = t.server
WHERE s.destroyed_at IS NULL AND t.payload IS NOT NULL`).all() as Array<{ payload: string }>;
return rows.map(row => JSON.parse(row.payload) as GameTelemetry);
const rows = db()
.prepare(
`SELECT t.payload FROM server_telemetry t JOIN servers s ON s.name = t.server
WHERE s.destroyed_at IS NULL AND t.payload IS NOT NULL`
)
.all() as Array<{ payload: string }>;
return rows.map((row) => JSON.parse(row.payload) as GameTelemetry);
}
export function saveTelemetry(report: GameTelemetry): GameTelemetry {
// Keep callers outside the HTTP route atomic too.
const handle = db();
return handle.transaction(() => {
const row = handle.prepare('SELECT payload FROM server_telemetry WHERE server = ?').get(report.server) as { payload: string | null } | undefined;
const previous: GameTelemetry | undefined = row?.payload ? JSON.parse(row.payload) : undefined;
const row = handle
.prepare('SELECT payload FROM server_telemetry WHERE server = ?')
.get(report.server) as { payload: string | null } | undefined;
const previous: GameTelemetry | undefined = row?.payload
? JSON.parse(row.payload)
: undefined;
recordLeaderboardTelemetry(handle, report);
// Record each occupancy transition at receipt, before a heartbeat can
// replace game_start or an empty report between status polls.
const after = latestGameSubmission(report.server);
if (!after || Date.parse(report.receivedAt) > Date.parse(after)) {
if (report.players.length > 0) {
handle.prepare('UPDATE servers SET active_since = COALESCE(active_since, ?), inactive_since = NULL WHERE name = ?')
handle
.prepare(
'UPDATE servers SET active_since = COALESCE(active_since, ?), inactive_since = NULL WHERE name = ?'
)
.run(report.receivedAt, report.server);
if (report.event === 'game_start') {
handle.prepare('UPDATE servers SET game_started_at = ?, game_ended_at = NULL WHERE name = ?').run(report.receivedAt, report.server);
handle
.prepare(
'UPDATE servers SET game_started_at = ?, game_ended_at = NULL WHERE name = ?'
)
.run(report.receivedAt, report.server);
} else if (report.event === 'game_end') {
handle.prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?').run(report.receivedAt, report.server);
handle
.prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?')
.run(report.receivedAt, report.server);
}
} else {
handle.prepare(`UPDATE servers SET active_since = NULL, inactive_since = COALESCE(inactive_since, ?),
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(report.receivedAt, report.server);
handle
.prepare(
`UPDATE servers SET active_since = NULL, inactive_since = COALESCE(inactive_since, ?),
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`
)
.run(report.receivedAt, report.server);
}
}
const serverTiming = handle.prepare(`SELECT active_since AS activeSince, inactive_since AS inactiveSince,
game_started_at AS gameStartedAt, game_ended_at AS gameEndedAt FROM servers WHERE name = ?`).get(report.server) as ServerTiming;
const match = handle.prepare('SELECT id FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1').get(report.server) as { id: number };
const serverTiming = handle
.prepare(
`SELECT active_since AS activeSince, inactive_since AS inactiveSince,
game_started_at AS gameStartedAt, game_ended_at AS gameEndedAt FROM servers WHERE name = ?`
)
.get(report.server) as ServerTiming;
const match = handle
.prepare('SELECT id FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1')
.get(report.server) as { id: number };
const sameMatch = previous?.timing?.matchId === match.id;
// A gap beyond the normal heartbeat window cannot establish continuity.
const continuous = previous && Date.parse(report.receivedAt) - Date.parse(previous.receivedAt) <= 120_000;
report = { ...report, timing: { matchId: match.id, server: serverTiming, players: Object.fromEntries(report.players.map(player => {
const prior = continuous ? previous?.timing?.players[player.guid] : undefined;
return [player.guid, {
serverJoinedAt: prior?.serverJoinedAt ?? report.receivedAt,
matchJoinedAt: sameMatch && prior ? prior.matchJoinedAt : report.receivedAt,
roundStarted: sameMatch && prior ? prior.roundStarted : report.event === 'game_start' ? 1 : report.round
}];
})) } };
handle.prepare(`INSERT INTO server_telemetry (server, round, players, received_at, payload)
const continuous =
previous && Date.parse(report.receivedAt) - Date.parse(previous.receivedAt) <= 120_000;
report = {
...report,
timing: {
matchId: match.id,
server: serverTiming,
players: Object.fromEntries(
report.players.map((player) => {
const prior = continuous
? previous?.timing?.players[player.guid]
: undefined;
return [
player.guid,
{
serverJoinedAt: prior?.serverJoinedAt ?? report.receivedAt,
matchJoinedAt:
sameMatch && prior ? prior.matchJoinedAt : report.receivedAt,
roundStarted:
sameMatch && prior
? prior.roundStarted
: report.event === 'game_start'
? 1
: report.round
}
];
})
)
}
};
handle
.prepare(
`INSERT INTO server_telemetry (server, round, players, received_at, payload)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(server) DO UPDATE SET
round = excluded.round, players = excluded.players, received_at = excluded.received_at, payload = excluded.payload`)
.run(report.server, report.event === 'game_start' ? 0 : report.round, report.players.length, report.receivedAt, JSON.stringify(report));
round = excluded.round, players = excluded.players, received_at = excluded.received_at, payload = excluded.payload`
)
.run(
report.server,
report.event === 'game_start' ? 0 : report.round,
report.players.length,
report.receivedAt,
JSON.stringify(report)
);
if (report.event === 'game_end') {
// Legacy betas also take match completion directly from the script.
if (report.round >= MIN_RECORDED_ROUND) handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND server_deleted_at IS NULL AND stage = 'awaiting-match'
AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`)
.run(report.receivedAt, report.server, report.receivedAt);
if (report.round >= MIN_RECORDED_ROUND)
handle
.prepare(
`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND server_deleted_at IS NULL AND stage = 'awaiting-match'
AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`
)
.run(report.receivedAt, report.server, report.receivedAt);
}
return report;
})();