josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/hooks.server.ts
+24 -6
View File
@@ -13,7 +13,7 @@ let rejectionLoggerRegistered = false;
function boot(): void {
if (booted) return;
if (!rejectionLoggerRegistered) {
process.on('unhandledRejection', reason => {
process.on('unhandledRejection', (reason) => {
console.error('[process] unhandled rejection:', reason);
});
rejectionLoggerRegistered = true;
@@ -30,13 +30,31 @@ function boot(): void {
export const handle: Handle = async (input) => {
boot();
const started = performance.now();
const response = await withHostRequestContext(() => withOperationSource(`${input.event.request.method} ${input.event.route.id ?? 'Web request'}`,
() => authenticationHandle(input)));
const response = await withHostRequestContext(() =>
withOperationSource(
`${input.event.request.method} ${input.event.route.id ?? 'Web request'}`,
() => authenticationHandle(input)
)
);
const elapsed = performance.now() - started;
response.headers.append('Server-Timing', `app;dur=${elapsed.toFixed(1)}`);
if (process.env.EC_PERF_LOG === 'true' && elapsed > 250 && !input.event.route.id?.startsWith('/api/agent/')) {
console.info('[request]', { route: input.event.route.id, method: input.event.request.method, status: response.status, elapsedMs: Math.round(elapsed) });
if (
process.env.EC_PERF_LOG === 'true' &&
elapsed > 250 &&
!input.event.route.id?.startsWith('/api/agent/')
) {
console.info('[request]', {
route: input.event.route.id,
method: input.event.request.method,
status: response.status,
elapsedMs: Math.round(elapsed)
});
}
if (input.event.locals.user && !['GET','HEAD','OPTIONS'].includes(input.event.request.method) && response.status < 400) invalidateFleet();
if (
input.event.locals.user &&
!['GET', 'HEAD', 'OPTIONS'].includes(input.event.request.method) &&
response.status < 400
)
invalidateFleet();
return response;
};
src/lib/activity.ts
+38 -11
View File
@@ -1,17 +1,37 @@
export const activityTypes = {
create: 'Create server', start: 'Start server', restart: 'Restart server', rebuild: 'Rebuild server',
stop: 'Stop server', remove: 'Destroy server', update: 'Game update', files: 'File changes',
health: 'Health finding', drift: 'File drift', host: 'Host operation', beta: 'Beta test'
create: 'Create server',
start: 'Start server',
restart: 'Restart server',
rebuild: 'Rebuild server',
stop: 'Stop server',
remove: 'Destroy server',
update: 'Game update',
files: 'File changes',
health: 'Health finding',
drift: 'File drift',
host: 'Host operation',
beta: 'Beta test'
} as const;
export type ActivityType = keyof typeof activityTypes;
export const activitySources = {
provisioning: 'Server operations', maintenance: 'Maintenance & recovery', recovery: 'Recovery',
campaign: 'Campaigns', beta: 'Betas', host: 'Hosts', autoscaling: 'Autoscaling'
provisioning: 'Server operations',
maintenance: 'Maintenance & recovery',
recovery: 'Recovery',
campaign: 'Campaigns',
beta: 'Betas',
host: 'Hosts',
autoscaling: 'Autoscaling'
} as const;
export type ActivitySource = keyof typeof activitySources;
export const activityStates = {
queued: 'Queued', running: 'In progress', succeeded: 'Completed', warning: 'Completed with warnings',
failed: 'Failed', cancelled: 'Cancelled', attention: 'Needs attention', unknown: 'Unknown'
queued: 'Queued',
running: 'In progress',
succeeded: 'Completed',
warning: 'Completed with warnings',
failed: 'Failed',
cancelled: 'Cancelled',
attention: 'Needs attention',
unknown: 'Unknown'
} as const;
export type ActivityState = keyof typeof activityStates;
@@ -54,8 +74,11 @@ export interface ActivityPage {
query: ActivityQuery;
}
function selection<T extends string>(value: string | null, labels: Record<T, string>): T | undefined {
return value !== null && Object.hasOwn(labels, value) ? value as T : undefined;
function selection<T extends string>(
value: string | null,
labels: Record<T, string>
): T | undefined {
return value !== null && Object.hasOwn(labels, value) ? (value as T) : undefined;
}
function integer(value: string | null, fallback: number, min: number, max: number): number {
if (value === null || !/^\d+$/.test(value)) return fallback;
@@ -72,7 +95,10 @@ export function parseActivityQuery(params: URLSearchParams): ActivityQuery {
if (params.get('excludeProvisioning') === 'true') query.excludeProvisioning = true;
if (params.get('excludeMaintenance') === 'true') query.excludeMaintenance = true;
for (const key of ['server', 'host', 'q'] as const) {
const value = params.get(key)?.trim().slice(0, key === 'q' ? 200 : 255);
const value = params
.get(key)
?.trim()
.slice(0, key === 'q' ? 200 : 255);
if (value) query[key] = value;
}
query.type = selection(params.get('type'), activityTypes);
@@ -86,6 +112,7 @@ export function parseActivityQuery(params: URLSearchParams): ActivityQuery {
export function normalizeActivityQuery(query: ActivityQuery): ActivityQuery {
const params = new URLSearchParams();
for (const [key, value] of Object.entries(query)) if (value !== undefined) params.set(key, String(value));
for (const [key, value] of Object.entries(query))
if (value !== undefined) params.set(key, String(value));
return parseActivityQuery(params);
}
src/lib/artifact-version.ts
+3 -1
View File
@@ -1,5 +1,7 @@
/** Keep the upload number visible: two builds may share a publisher's release tag. */
export function artifactVersionLabel(version: { version: number; release_version?: string | null } | null | undefined): string {
export function artifactVersionLabel(
version: { version: number; release_version?: string | null } | null | undefined
): string {
if (!version) return '—';
return version.release_version
? `${version.release_version} (upload ${version.version})`
src/lib/auth.ts
+3 -1
View File
@@ -16,5 +16,7 @@ export interface AuthUser {
}
export const roleLabels: Record<UserRole, string> = {
admin: 'Administrator', operator: 'Operator', viewer: 'Viewer'
admin: 'Administrator',
operator: 'Operator',
viewer: 'Viewer'
};
src/lib/autoscaling.ts
+95 -23
View File
@@ -1,56 +1,128 @@
import { zombieMapBySlug, type ZombieGame } from './map-aliases.js';
export const autoscalingModes = ['off', 'create-destroy'] as const;
export type AutoscalingMode = typeof autoscalingModes[number];
export type AutoscalingMode = (typeof autoscalingModes)[number];
export interface ScalingSettings {
game: ZombieGame;
/** '*' applies the targets independently to every map. */
mapSlug: string;
mode: AutoscalingMode;
minimum: number; maximum: number; spares: number; emptyMinutes: number;
minimum: number;
maximum: number;
spares: number;
emptyMinutes: number;
}
export interface ScalingPolicy extends ScalingSettings {
id: string; revision: number; pausedReason: string | null; updatedAt: string;
id: string;
revision: number;
pausedReason: string | null;
updatedAt: string;
}
export interface ScalingMember {
name: string; mapSlug?: string; identity: string; state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected';
players: number | null; observedAt: number | null; source: string | null;
emptySince: number | null; blocker: string | null; operationId: string | null;
name: string;
mapSlug?: string;
identity: string;
state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected';
players: number | null;
observedAt: number | null;
source: string | null;
emptySince: number | null;
blocker: string | null;
operationId: string | null;
emptyStartedAt?: number | null;
}
export interface ScalingSnapshot {
members: ScalingMember[]; reservedNames: string[]; keyPool: { total: number; available: number }; blockers: string[];
members: ScalingMember[];
reservedNames: string[];
keyPool: { total: number; available: number };
blockers: string[];
}
export interface ScalingDecision {
action: 'create' | 'destroy' | 'wait'; reason: string; server: string | null;
desired: number; deficit: number; occupied: number; empty: number; pending: number; unknown: number;
action: 'create' | 'destroy' | 'wait';
reason: string;
server: string | null;
desired: number;
deficit: number;
occupied: number;
empty: number;
pending: number;
unknown: number;
}
export interface ScalingEvaluation {
policy: ScalingPolicy;
snapshot: ScalingSnapshot;
decision: ScalingDecision;
blockers: string[];
}
export interface ScalingHistory {
id: string;
at: number;
lastSeenAt: number;
decision: ScalingDecision;
operationId: string | null;
operationState: string | null;
}
export interface ScalingEvaluation { policy: ScalingPolicy; snapshot: ScalingSnapshot; decision: ScalingDecision; blockers: string[] }
export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null; operationState: string | null }
export interface ScalingView extends ScalingEvaluation {
history: ScalingHistory[]; lastEvaluatedAt: number | null;
totals: { observedMinutes: number; emptyLobbyHours: number; deficitMinutes: number; unknownMinutes: number };
operations: { creates: number; removals: number; cancelled: number; failed: number; readinessSamples: number; readinessMeanSeconds: number | null; readinessMaxSeconds: number | null };
history: ScalingHistory[];
lastEvaluatedAt: number | null;
totals: {
observedMinutes: number;
emptyLobbyHours: number;
deficitMinutes: number;
unknownMinutes: number;
};
operations: {
creates: number;
removals: number;
cancelled: number;
failed: number;
readinessSamples: number;
readinessMeanSeconds: number | null;
readinessMaxSeconds: number | null;
};
ownsMap: boolean;
unsettled: { id: string; action: string; state: string; stage: string }[];
}
export interface ScalingReport { policies: ScalingView[]; globalPaused: boolean; liveAvailable: boolean; removalAvailable: boolean }
export interface ScalingReport {
policies: ScalingView[];
globalPaused: boolean;
liveAvailable: boolean;
removalAvailable: boolean;
}
export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-destroy';
export function scalingSettings(input: unknown): ScalingSettings {
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide an autoscaling policy.');
if (!input || typeof input !== 'object' || Array.isArray(input))
throw new Error('Provide an autoscaling policy.');
const value = input as Record<string, unknown>;
const { game, mapSlug } = value;
if ((game !== 't5' && game !== 't6') || typeof mapSlug !== 'string' || (mapSlug !== '*' && zombieMapBySlug(game, mapSlug)?.slug !== mapSlug)) throw new Error('Choose a supported game and map variant, or all maps.');
if (
(game !== 't5' && game !== 't6') ||
typeof mapSlug !== 'string' ||
(mapSlug !== '*' && zombieMapBySlug(game, mapSlug)?.slug !== mapSlug)
)
throw new Error('Choose a supported game and map variant, or all maps.');
const mode = value.mode ?? 'off';
if (!autoscalingModes.includes(mode as AutoscalingMode)) throw new Error('Choose a valid autoscaling mode.');
if (!autoscalingModes.includes(mode as AutoscalingMode))
throw new Error('Choose a valid autoscaling mode.');
const integer = (key: string, fallback: number | undefined, max: number) => {
const n = value[key] ?? fallback;
if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > max) throw new Error(`${key} must be an integer from 1 to ${max}.`);
if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > max)
throw new Error(`${key} must be an integer from 1 to ${max}.`);
return n;
};
const minimum = integer('minimum', 1, 20), maximum = integer('maximum', undefined, 20), spares = integer('spares', 1, 20);
if (minimum > maximum || spares > maximum) throw new Error('Minimum and spare target cannot exceed the maximum.');
return { game, mapSlug, mode: mode as AutoscalingMode, minimum, maximum, spares,
emptyMinutes: integer('emptyMinutes', 15, 1440) };
const minimum = integer('minimum', 1, 20),
maximum = integer('maximum', undefined, 20),
spares = integer('spares', 1, 20);
if (minimum > maximum || spares > maximum)
throw new Error('Minimum and spare target cannot exceed the maximum.');
return {
game,
mapSlug,
mode: mode as AutoscalingMode,
minimum,
maximum,
spares,
emptyMinutes: integer('emptyMinutes', 15, 1440)
};
}
src/lib/beta-draft.ts
+120 -34
View File
@@ -2,9 +2,12 @@ import { requirementKinds, type BetaPlan } from './beta-plan.js';
export const betaDraftKey = 'electric-cherry:beta-draft:v1';
export interface BetaDraft {
version: 1; savedAt: string; step: number;
version: 1;
savedAt: string;
step: number;
picks: Array<{ scriptId: number | null; versionId: number | null }>;
servers: string[]; testPlan: BetaPlan;
servers: string[];
testPlan: BetaPlan;
selectionMode: 'automatic' | 'manual';
instanceCount: number | null;
label?: string;
@@ -14,55 +17,138 @@ export function readBetaDraft(raw: string | null): BetaDraft | null {
if (!raw) return null;
try {
const d = JSON.parse(raw);
if (d?.version !== 1 || !Array.isArray(d.picks) || !d.picks.length || d.picks.length > 100
|| !d.picks.every((p: BetaDraft['picks'][number]) => p && [p.scriptId, p.versionId].every(v => v === null || Number.isSafeInteger(v) && v! > 0))
|| !Array.isArray(d.servers) || !d.servers.every((s: unknown) => typeof s === 'string')
|| d.testPlan?.version !== 1 || !Array.isArray(d.testPlan.requirements) || d.testPlan.requirements.length > 30
|| !d.testPlan.requirements.every((r: BetaPlan['requirements'][number]) => r && requirementKinds.includes(r.kind) && typeof r.id === 'string' && /^[a-zA-Z0-9_-]{1,64}$/.test(r.id) && typeof r.label === 'string')) return null;
if (d.selectionMode !== undefined && !['automatic', 'manual'].includes(d.selectionMode)) return null;
if (d.instancesPerGame !== undefined && (!d.instancesPerGame || typeof d.instancesPerGame !== 'object'
|| Array.isArray(d.instancesPerGame) || !Object.entries(d.instancesPerGame).every(([game, count]) =>
['t5', 't6'].includes(game) && (count === null || typeof count === 'number' && Number.isFinite(count))))) return null;
if (d.instanceCount !== undefined && d.instanceCount !== null
&& (typeof d.instanceCount !== 'number' || !Number.isFinite(d.instanceCount))) return null;
if (
d?.version !== 1 ||
!Array.isArray(d.picks) ||
!d.picks.length ||
d.picks.length > 100 ||
!d.picks.every(
(p: BetaDraft['picks'][number]) =>
p &&
[p.scriptId, p.versionId].every(
(v) => v === null || (Number.isSafeInteger(v) && v! > 0)
)
) ||
!Array.isArray(d.servers) ||
!d.servers.every((s: unknown) => typeof s === 'string') ||
d.testPlan?.version !== 1 ||
!Array.isArray(d.testPlan.requirements) ||
d.testPlan.requirements.length > 30 ||
!d.testPlan.requirements.every(
(r: BetaPlan['requirements'][number]) =>
r &&
requirementKinds.includes(r.kind) &&
typeof r.id === 'string' &&
/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) &&
typeof r.label === 'string'
)
)
return null;
if (d.selectionMode !== undefined && !['automatic', 'manual'].includes(d.selectionMode))
return null;
if (
d.instancesPerGame !== undefined &&
(!d.instancesPerGame ||
typeof d.instancesPerGame !== 'object' ||
Array.isArray(d.instancesPerGame) ||
!Object.entries(d.instancesPerGame).every(
([game, count]) =>
['t5', 't6'].includes(game) &&
(count === null || (typeof count === 'number' && Number.isFinite(count)))
))
)
return null;
if (
d.instanceCount !== undefined &&
d.instanceCount !== null &&
(typeof d.instanceCount !== 'number' || !Number.isFinite(d.instanceCount))
)
return null;
const previousCounts = Object.values(d.instancesPerGame ?? {}) as Array<number | null>;
const instanceCount = d.instanceCount !== undefined ? d.instanceCount
: previousCounts.includes(null) ? null : previousCounts.length ? Math.max(...previousCounts as number[]) : 1;
const instanceCount =
d.instanceCount !== undefined
? d.instanceCount
: previousCounts.includes(null)
? null
: previousCounts.length
? Math.max(...(previousCounts as number[]))
: 1;
// Existing drafts represent explicit selections and must not silently switch modes.
const selectionMode = d.selectionMode ?? (d.servers.length ? 'manual' : 'automatic');
// Revisit server setup when old per-game counts cannot be retained as one value.
const changedCounts = selectionMode === 'automatic' && d.instanceCount === undefined && new Set(previousCounts).size > 1;
return { ...d, selectionMode,
instanceCount, step: Math.max(0, Math.min(changedCounts ? 1 : 3, Number(d.step) || 0)) };
} catch { return null; }
const changedCounts =
selectionMode === 'automatic' &&
d.instanceCount === undefined &&
new Set(previousCounts).size > 1;
return {
...d,
selectionMode,
instanceCount,
step: Math.max(0, Math.min(changedCounts ? 1 : 3, Number(d.step) || 0))
};
} catch {
return null;
}
}
interface BetaFileScope { scope_type: 'game' | 'map' | null; scope_value: string | null }
interface BetaServerScope { name: string; game: string | null; map: string | null }
interface BetaFileScope {
scope_type: 'game' | 'map' | null;
scope_value: string | null;
}
interface BetaServerScope {
name: string;
game: string | null;
map: string | null;
}
export function betaScopeMatches(file: BetaFileScope, server: BetaServerScope): boolean {
return !file.scope_type || (file.scope_type === 'game' ? server.game : server.map) === file.scope_value;
return (
!file.scope_type ||
(file.scope_type === 'game' ? server.game : server.map) === file.scope_value
);
}
/** A bundle installs each file only on its own matching participants. */
export function betaCandidateServers<T extends BetaServerScope>(files: readonly BetaFileScope[], servers: readonly T[]): T[] {
return servers.filter(server => files.some(file => betaScopeMatches(file, server)));
export function betaCandidateServers<T extends BetaServerScope>(
files: readonly BetaFileScope[],
servers: readonly T[]
): T[] {
return servers.filter((server) => files.some((file) => betaScopeMatches(file, server)));
}
export function betaApplicableGames(files: readonly BetaFileScope[], servers: readonly BetaServerScope[]): string[] {
return [...new Set([
...files.filter(file => file.scope_type === 'game').map(file => file.scope_value),
...betaCandidateServers(files, servers).map(server => server.game)
].filter((game): game is string => game === 't5' || game === 't6'))].sort();
export function betaApplicableGames(
files: readonly BetaFileScope[],
servers: readonly BetaServerScope[]
): string[] {
return [
...new Set(
[
...files
.filter((file) => file.scope_type === 'game')
.map((file) => file.scope_value),
...betaCandidateServers(files, servers).map((server) => server.game)
].filter((game): game is string => game === 't5' || game === 't6')
)
].sort();
}
/** Expand the shared count only to games represented by the selected files. */
export function betaInstanceCounts(games: readonly string[], count: number | null): Record<string, number> {
export function betaInstanceCounts(
games: readonly string[],
count: number | null
): Record<string, number> {
if (count === null || !Number.isSafeInteger(count) || count < 1 || count > 1000) return {};
return Object.fromEntries(games.map(game => [game, count]));
return Object.fromEntries(games.map((game) => [game, count]));
}
export function compatibleBetaServers(selected: Iterable<string>, candidates: readonly string[]): { kept: Set<string>; removed: string[] } {
const allowed = new Set(candidates), names = [...selected];
return { kept: new Set(names.filter(name => allowed.has(name))), removed: names.filter(name => !allowed.has(name)) };
export function compatibleBetaServers(
selected: Iterable<string>,
candidates: readonly string[]
): { kept: Set<string>; removed: string[] } {
const allowed = new Set(candidates),
names = [...selected];
return {
kept: new Set(names.filter((name) => allowed.has(name))),
removed: names.filter((name) => !allowed.has(name))
};
}
src/lib/beta-evidence.ts
+43 -18
View File
@@ -1,23 +1,35 @@
import type { BetaRequirement, BetaTestView } from './beta-plan.js';
import { needsGameTelemetry } from './beta-plan-summary.js';
export function betaWaitingCondition(requirement: BetaRequirement, observation?: BetaTestView['servers'][number]['observation']): string {
if (requirement.kind === 'manual') return 'Waiting for a tester to record their name and observation.';
if (requirement.kind === 'recovery') return 'The other checks must pass before the next restart when empty; each cycle needs fresh evidence.';
if (requirement.kind === 'healthy-minutes') return observation?.healthy === true
? 'Collecting consecutive fresh healthy observations; an outage or monitoring gap resets this duration.'
: 'Waiting for fresh container and RCON health observations. Unhealthy or stale observations reset this duration.';
if (requirement.kind === 'failure-budget') return 'An observed health outage exceeded the budget. Investigate before starting a fresh attempt.';
export function betaWaitingCondition(
requirement: BetaRequirement,
observation?: BetaTestView['servers'][number]['observation']
): string {
if (requirement.kind === 'manual')
return 'Waiting for a tester to record their name and observation.';
if (requirement.kind === 'recovery')
return 'The other checks must pass before the next restart when empty; each cycle needs fresh evidence.';
if (requirement.kind === 'healthy-minutes')
return observation?.healthy === true
? 'Collecting consecutive fresh healthy observations; an outage or monitoring gap resets this duration.'
: 'Waiting for fresh container and RCON health observations. Unhealthy or stale observations reset this duration.';
if (requirement.kind === 'failure-budget')
return 'An observed health outage exceeded the budget. Investigate before starting a fresh attempt.';
if (needsGameTelemetry(requirement)) {
if (!observation?.lastGameplayAt) return 'No gameplay report received for this attempt. Check the game reporter and Telemetry API requests.';
if (!observation?.lastGameplayAt)
return 'No gameplay report received for this attempt. Check the game reporter and Telemetry API requests.';
if (observation.gameEnded) return 'Waiting for the next game and its qualifying reports.';
if (requirement.kind === 'population') return `Waiting for a game report containing at least ${requirement.target} players.`;
if (requirement.kind === 'population')
return `Waiting for a game report containing at least ${requirement.target} players.`;
const players = `at least ${requirement.minPlayers} player${requirement.minPlayers === 1 ? '' : 's'}`;
if ((observation.players ?? 0) < requirement.minPlayers) return `The latest report has ${observation.players ?? 0} players; this check needs ${players}.`;
if (requirement.kind === 'matches') return requirement.minMinutes && !observation.matchStarted
? 'This match has no game-start report, so its duration cannot qualify. Waiting for a new game start and game end.'
: `Waiting for game end${requirement.minRound ? ` after reaching round ${requirement.minRound}` : ''} with ${players}${requirement.minMinutes ? ` and at least ${requirement.minMinutes} minutes since game start` : ''}.`;
if (requirement.kind === 'round') return `Waiting for a round change, heartbeat, or game end reporting round ${requirement.target} or higher with ${players}.`;
if ((observation.players ?? 0) < requirement.minPlayers)
return `The latest report has ${observation.players ?? 0} players; this check needs ${players}.`;
if (requirement.kind === 'matches')
return requirement.minMinutes && !observation.matchStarted
? 'This match has no game-start report, so its duration cannot qualify. Waiting for a new game start and game end.'
: `Waiting for game end${requirement.minRound ? ` after reaching round ${requirement.minRound}` : ''} with ${players}${requirement.minMinutes ? ` and at least ${requirement.minMinutes} minutes since game start` : ''}.`;
if (requirement.kind === 'round')
return `Waiting for a round change, heartbeat, or game end reporting round ${requirement.target} or higher with ${players}.`;
return `Time is credited when the next report arrives, with ${players} at both ends of the interval. It does not advance continuously between reports.`;
}
return 'Waiting for qualifying evidence.';
@@ -25,13 +37,26 @@ export function betaWaitingCondition(requirement: BetaRequirement, observation?:
export function betaEvidenceSummary(event: BetaTestView['evidence'][number]): string {
let payload: Record<string, unknown>;
try { payload = JSON.parse(event.payload); } catch { return event.kind.replaceAll('-', ' ').replaceAll('_', ' '); }
try {
payload = JSON.parse(event.payload);
} catch {
return event.kind.replaceAll('-', ' ').replaceAll('_', ' ');
}
if (!payload || typeof payload !== 'object') return event.kind;
if (['game_start', 'round_change', 'heartbeat', 'game_end'].includes(event.kind)) {
const label = event.kind === 'game_start' ? 'Game started' : event.kind === 'game_end' ? 'Game ended' : event.kind === 'heartbeat' ? 'Heartbeat' : 'Round changed';
const label =
event.kind === 'game_start'
? 'Game started'
: event.kind === 'game_end'
? 'Game ended'
: event.kind === 'heartbeat'
? 'Heartbeat'
: 'Round changed';
return `${label}${typeof payload.round === 'number' ? ` at round ${payload.round}` : ''}${Array.isArray(payload.players) ? ` with ${payload.players.length} players` : ''}${typeof payload.map === 'string' ? ` on ${payload.map}` : ''}.`;
}
if (event.kind === 'manual-sign-off') return `${payload.requirement ?? 'Checklist'}: ${payload.note ?? 'Signed off'}`;
if (event.kind === 'health' || event.kind === 'status-sample') return `${payload.healthy ? 'Healthy observation' : 'Unhealthy observation'}${payload.reason ? `: ${payload.reason}` : '.'}`;
if (event.kind === 'manual-sign-off')
return `${payload.requirement ?? 'Checklist'}: ${payload.note ?? 'Signed off'}`;
if (event.kind === 'health' || event.kind === 'status-sample')
return `${payload.healthy ? 'Healthy observation' : 'Unhealthy observation'}${payload.reason ? `: ${payload.reason}` : '.'}`;
return event.kind.replaceAll('-', ' ').replaceAll('_', ' ');
}
src/lib/beta-plan-summary.ts
+48 -16
View File
@@ -1,34 +1,66 @@
import type { BetaPlan, BetaRequirement } from './beta-plan.js';
const count = (n: number, word: string) => `${n} ${word}${n === 1 ? '' : 's'}`;
const duration = (minutes: number) => minutes >= 60 && minutes % 60 === 0 ? count(minutes / 60, 'hour') : count(minutes, 'minute');
const duration = (minutes: number) =>
minutes >= 60 && minutes % 60 === 0 ? count(minutes / 60, 'hour') : count(minutes, 'minute');
export const presetDescriptions: Record<string, string> = {
'Quick smoke test': 'A quick check that the server starts and stays healthy.',
'Gameplay validation': 'Prove the build through real matches and populated playtime.',
'Long soak': 'Exercise stability and participation over a longer test.'
};
export function requirementSummary(r: BetaRequirement): string {
const players = r.minPlayers > 1 ? ` with at least ${r.minPlayers} players` : ' with players present';
const players =
r.minPlayers > 1 ? ` with at least ${r.minPlayers} players` : ' with players present';
switch (r.kind) {
case 'round': return `Reach round ${r.target}${players}`;
case 'matches': return `Complete ${count(r.target, 'match').replace('matchs', 'matches')}${r.minRound ? ` reaching round ${r.minRound}` : ''}${r.minPlayers > 1 ? players : ''}${r.minMinutes ? ` lasting at least ${duration(r.minMinutes)} each` : ''}`;
case 'populated-minutes': return `Accumulate ${duration(r.target)}${players}`;
case 'player-hours': return `Accumulate ${r.target} player-hours${r.minPlayers > 1 ? players : ''}`;
case 'population': return `Reach ${count(r.target, 'player')} online`;
case 'sustained-population': return `Keep at least ${count(r.minPlayers, 'player')} online for ${duration(r.target)}`;
case 'healthy-minutes': return `Stay continuously healthy for ${duration(r.target)}`;
case 'failure-budget': return r.target === 0 ? 'Allow no health outages' : `Allow at most ${count(r.target, 'health outage')}`;
case 'recovery': return `Repeat the checks after ${count(r.target, 'recovery restart')}`;
case 'manual': return r.label;
case 'round':
return `Reach round ${r.target}${players}`;
case 'matches':
return `Complete ${count(r.target, 'match').replace('matchs', 'matches')}${r.minRound ? ` reaching round ${r.minRound}` : ''}${r.minPlayers > 1 ? players : ''}${r.minMinutes ? ` lasting at least ${duration(r.minMinutes)} each` : ''}`;
case 'populated-minutes':
return `Accumulate ${duration(r.target)}${players}`;
case 'player-hours':
return `Accumulate ${r.target} player-hours${r.minPlayers > 1 ? players : ''}`;
case 'population':
return `Reach ${count(r.target, 'player')} online`;
case 'sustained-population':
return `Keep at least ${count(r.minPlayers, 'player')} online for ${duration(r.target)}`;
case 'healthy-minutes':
return `Stay continuously healthy for ${duration(r.target)}`;
case 'failure-budget':
return r.target === 0
? 'Allow no health outages'
: `Allow at most ${count(r.target, 'health outage')}`;
case 'recovery':
return `Repeat the checks after ${count(r.target, 'recovery restart')}`;
case 'manual':
return r.label;
}
}
export function coverageSummary(plan: BetaPlan): string {
return plan.scope === 'all' ? 'Every selected server' : plan.scope === 'quorum' ? `At least ${count(plan.quorum, 'selected server')}`
: plan.scope === 'map' ? 'One server per selected map' : plan.scope === 'game' ? 'One server per selected game' : 'One server per game / map';
return plan.scope === 'all'
? 'Every selected server'
: plan.scope === 'quorum'
? `At least ${count(plan.quorum, 'selected server')}`
: plan.scope === 'map'
? 'One server per selected map'
: plan.scope === 'game'
? 'One server per selected game'
: 'One server per game / map';
}
export function needsGameTelemetry(r: BetaRequirement): boolean {
return ['round', 'matches', 'population', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind);
return [
'round',
'matches',
'population',
'populated-minutes',
'player-hours',
'sustained-population'
].includes(r.kind);
}
export function requirementCategory(r: BetaRequirement): string {
return r.kind === 'manual' ? 'Tester sign-off' : ['healthy-minutes', 'failure-budget'].includes(r.kind) ? 'Continuous check' : 'Milestone';
return r.kind === 'manual'
? 'Tester sign-off'
: ['healthy-minutes', 'failure-budget'].includes(r.kind)
? 'Continuous check'
: 'Milestone';
}
src/lib/beta-plan.ts
+282 -56
View File
@@ -1,6 +1,16 @@
export const requirementKinds = ['round', 'matches', 'populated-minutes', 'player-hours', 'population',
'sustained-population', 'healthy-minutes', 'failure-budget', 'recovery', 'manual'] as const;
export type RequirementKind = typeof requirementKinds[number];
export const requirementKinds = [
'round',
'matches',
'populated-minutes',
'player-hours',
'population',
'sustained-population',
'healthy-minutes',
'failure-budget',
'recovery',
'manual'
] as const;
export type RequirementKind = (typeof requirementKinds)[number];
export interface BetaRequirement {
id: string;
kind: RequirementKind;
@@ -21,90 +31,306 @@ export interface BetaPlan {
waveSize: number; // 0 deploys all selected servers; otherwise pass each wave before expanding.
}
export const requirementLabels: Record<RequirementKind, string> = {
round: 'Reach round', matches: 'Complete qualifying matches', 'populated-minutes': 'Populated minutes',
'player-hours': 'Player-hours', population: 'Reach player count', 'sustained-population': 'Sustained population (minutes)',
'healthy-minutes': 'Continuous healthy minutes', 'failure-budget': 'Maximum health outages',
recovery: 'Successful recovery cycles', manual: 'Tester checklist'
round: 'Reach round',
matches: 'Complete qualifying matches',
'populated-minutes': 'Populated minutes',
'player-hours': 'Player-hours',
population: 'Reach player count',
'sustained-population': 'Sustained population (minutes)',
'healthy-minutes': 'Continuous healthy minutes',
'failure-budget': 'Maximum health outages',
recovery: 'Successful recovery cycles',
manual: 'Tester checklist'
};
export function requirement(kind: RequirementKind, id: string): BetaRequirement {
return { id, kind, label: requirementLabels[kind], target: kind === 'round' ? 25 : kind === 'failure-budget' ? 0 : 1,
minRound: 0, minPlayers: 1, minMinutes: 0 };
return {
id,
kind,
label: requirementLabels[kind],
target: kind === 'round' ? 25 : kind === 'failure-budget' ? 0 : 1,
minRound: 0,
minPlayers: 1,
minMinutes: 0
};
}
export function defaultPlan(): BetaPlan {
return { version: 1, requirements: [requirement('healthy-minutes', 'health')], scope: 'all', quorum: 1,
timeoutHours: 24, failureAction: 'fail', rollbackRestart: 'when-empty', waveSize: 0 };
return {
version: 1,
requirements: [requirement('healthy-minutes', 'health')],
scope: 'all',
quorum: 1,
timeoutHours: 24,
failureAction: 'fail',
rollbackRestart: 'when-empty',
waveSize: 0
};
}
export const builtInPlans: Array<{ name: string; plan: BetaPlan }> = [
{ name: 'Quick smoke test', plan: defaultPlan() },
{ name: 'Gameplay validation', plan: { ...defaultPlan(), requirements: [
{ ...requirement('matches', 'matches'), target: 2, minRound: 25 },
{ ...requirement('populated-minutes', 'playtime'), target: 90 }, requirement('failure-budget', 'outages')
] } },
{ name: 'Long soak', plan: { ...defaultPlan(), timeoutHours: 72, requirements: [
{ ...requirement('healthy-minutes', 'health'), target: 360 },
{ ...requirement('player-hours', 'participation'), target: 10 }, requirement('failure-budget', 'outages')
] } }
{
name: 'Gameplay validation',
plan: {
...defaultPlan(),
requirements: [
{ ...requirement('matches', 'matches'), target: 2, minRound: 25 },
{ ...requirement('populated-minutes', 'playtime'), target: 90 },
requirement('failure-budget', 'outages')
]
}
},
{
name: 'Long soak',
plan: {
...defaultPlan(),
timeoutHours: 72,
requirements: [
{ ...requirement('healthy-minutes', 'health'), target: 360 },
{ ...requirement('player-hours', 'participation'), target: 10 },
requirement('failure-budget', 'outages')
]
}
}
];
/** Strict allow-list validation at every boundary (including stored presets). */
export function parseBetaPlan(value: unknown): BetaPlan {
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Test plan must be an object');
if (!value || typeof value !== 'object' || Array.isArray(value))
throw new Error('Test plan must be an object');
const p = value as BetaPlan;
if (p.version !== 1 || !['all', 'quorum', 'map', 'game', 'map-game'].includes(p.scope)) throw new Error('Invalid test plan version or scope');
if (!['fail', 'pause', 'rollback'].includes(p.failureAction) || !['manual', 'when-empty'].includes(p.rollbackRestart)) throw new Error('Invalid failure policy');
for (const [key, min, max] of [['quorum', 1, 1000], ['timeoutHours', 0.01, 8760], ['waveSize', 0, 1000]] as const) {
if (typeof p[key] !== 'number' || !Number.isFinite(p[key]) || p[key] < min || p[key] > max) throw new Error(`Invalid ${key}`);
if (p.version !== 1 || !['all', 'quorum', 'map', 'game', 'map-game'].includes(p.scope))
throw new Error('Invalid test plan version or scope');
if (
!['fail', 'pause', 'rollback'].includes(p.failureAction) ||
!['manual', 'when-empty'].includes(p.rollbackRestart)
)
throw new Error('Invalid failure policy');
for (const [key, min, max] of [
['quorum', 1, 1000],
['timeoutHours', 0.01, 8760],
['waveSize', 0, 1000]
] as const) {
if (typeof p[key] !== 'number' || !Number.isFinite(p[key]) || p[key] < min || p[key] > max)
throw new Error(`Invalid ${key}`);
}
if (!Number.isInteger(p.quorum) || !Number.isInteger(p.waveSize)) throw new Error('Quorum and wave size must be integers');
if (!Array.isArray(p.requirements) || !p.requirements.length || p.requirements.length > 30) throw new Error('Choose 1–30 requirements');
if (!Number.isInteger(p.quorum) || !Number.isInteger(p.waveSize))
throw new Error('Quorum and wave size must be integers');
if (!Array.isArray(p.requirements) || !p.requirements.length || p.requirements.length > 30)
throw new Error('Choose 1–30 requirements');
const ids = new Set<string>();
const requirements = p.requirements.map(r => {
if (!r || typeof r !== 'object' || !requirementKinds.includes(r.kind)) throw new Error('Unknown requirement');
if (typeof r.id !== 'string' || !/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) || r.id in Object.prototype || ids.has(r.id)) throw new Error('Requirement IDs must be unique, non-reserved identifiers');
const requirements = p.requirements.map((r) => {
if (!r || typeof r !== 'object' || !requirementKinds.includes(r.kind))
throw new Error('Unknown requirement');
if (
typeof r.id !== 'string' ||
!/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) ||
r.id in Object.prototype ||
ids.has(r.id)
)
throw new Error('Requirement IDs must be unique, non-reserved identifiers');
ids.add(r.id);
if (typeof r.label !== 'string' || !r.label.trim() || r.label.length > 200) throw new Error('Requirement needs a label (up to 200 characters)');
if (typeof r.label !== 'string' || !r.label.trim() || r.label.length > 200)
throw new Error('Requirement needs a label (up to 200 characters)');
for (const key of ['target', 'minRound', 'minPlayers', 'minMinutes'] as const) {
if (typeof r[key] !== 'number' || !Number.isFinite(r[key]) || r[key] < 0 || r[key] > 1_000_000) throw new Error(`Invalid requirement ${key}`);
if (
typeof r[key] !== 'number' ||
!Number.isFinite(r[key]) ||
r[key] < 0 ||
r[key] > 1_000_000
)
throw new Error(`Invalid requirement ${key}`);
}
if (r.kind !== 'failure-budget' && r.target <= 0) throw new Error('Requirement target must be positive');
if (['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(r.kind) && !Number.isInteger(r.target)) throw new Error('Count targets must be integers');
if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers)) throw new Error('Round and player qualifiers must be integers');
if (r.kind === 'manual' && r.target !== 1) throw new Error('Add a separate requirement for each checklist item');
return { id: r.id, kind: r.kind, label: r.label.trim(), target: r.target, minRound: r.minRound,
minPlayers: r.minPlayers, minMinutes: r.minMinutes };
if (r.kind !== 'failure-budget' && r.target <= 0)
throw new Error('Requirement target must be positive');
if (
['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(
r.kind
) &&
!Number.isInteger(r.target)
)
throw new Error('Count targets must be integers');
if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers))
throw new Error('Round and player qualifiers must be integers');
if (r.kind === 'manual' && r.target !== 1)
throw new Error('Add a separate requirement for each checklist item');
return {
id: r.id,
kind: r.kind,
label: r.label.trim(),
target: r.target,
minRound: r.minRound,
minPlayers: r.minPlayers,
minMinutes: r.minMinutes
};
});
return { version: 1, requirements, scope: p.scope, quorum: p.quorum, timeoutHours: p.timeoutHours,
failureAction: p.failureAction, rollbackRestart: p.rollbackRestart, waveSize: p.waveSize };
return {
version: 1,
requirements,
scope: p.scope,
quorum: p.quorum,
timeoutHours: p.timeoutHours,
failureAction: p.failureAction,
rollbackRestart: p.rollbackRestart,
waveSize: p.waveSize
};
}
export interface RequirementProgress { id: string; label: string; current: number; target: number; passed: boolean; failed: boolean; detail: string }
export interface RequirementProgress {
id: string;
label: string;
current: number;
target: number;
passed: boolean;
failed: boolean;
detail: string;
}
export interface BetaTestView {
plan: BetaPlan;
outcome: string;
reason: string | null;
servers: Array<{ server: string; server_deleted_at: string | null; phase: number; passed: boolean; progress: RequirementProgress[]; observingAt?: string | null;
observation?: { lastGameplayAt: string | null; lastEvent: string | null; players: number | null; round: number | null; matchStarted: boolean; gameEnded: boolean; healthy: boolean | null; healthCheckedAt: string | null } }>;
evidence: Array<{ id: number; server: string; server_deleted_at: string | null; received_at: string; kind: string; payload: string }>;
servers: Array<{
server: string;
server_deleted_at: string | null;
phase: number;
passed: boolean;
progress: RequirementProgress[];
observingAt?: string | null;
observation?: {
lastGameplayAt: string | null;
lastEvent: string | null;
players: number | null;
round: number | null;
matchStarted: boolean;
gameEnded: boolean;
healthy: boolean | null;
healthCheckedAt: string | null;
};
}>;
evidence: Array<{
id: number;
server: string;
server_deleted_at: string | null;
received_at: string;
kind: string;
payload: string;
}>;
}
export interface BetaPlanIssue { field: string; message: string; requirementId?: string }
export interface BetaPlanIssue {
field: string;
message: string;
requirementId?: string;
}
export function betaPlanIssues(plan: BetaPlan, serverCount?: number): BetaPlanIssue[] {
const issues: BetaPlanIssue[] = [];
const check = (field: string, value: number, min: number, max: number, integer: boolean, message: string, requirementId?: string) => {
if (typeof value !== 'number' || !Number.isFinite(value) || value < min || value > max || (integer && !Number.isInteger(value))) issues.push({ field, message, requirementId });
const check = (
field: string,
value: number,
min: number,
max: number,
integer: boolean,
message: string,
requirementId?: string
) => {
if (
typeof value !== 'number' ||
!Number.isFinite(value) ||
value < min ||
value > max ||
(integer && !Number.isInteger(value))
)
issues.push({ field, message, requirementId });
};
check('timeoutHours', plan.timeoutHours, .01, 8760, false, 'Enter a testing window between 0.01 and 8,760 hours.');
check('waveSize', plan.waveSize, 0, 1000, true, 'Enter a whole number from 0 to 1,000; 0 deploys to all selected servers.');
if (plan.scope === 'quorum') check('quorum', plan.quorum, 1, serverCount ?? 1000, true, `Choose between 1 and ${serverCount ?? 1000} qualifying servers.`);
if (!plan.requirements.length) issues.push({ field: 'requirements', message: 'Add at least one requirement to continue.' });
check(
'timeoutHours',
plan.timeoutHours,
0.01,
8760,
false,
'Enter a testing window between 0.01 and 8,760 hours.'
);
check(
'waveSize',
plan.waveSize,
0,
1000,
true,
'Enter a whole number from 0 to 1,000; 0 deploys to all selected servers.'
);
if (plan.scope === 'quorum')
check(
'quorum',
plan.quorum,
1,
serverCount ?? 1000,
true,
`Choose between 1 and ${serverCount ?? 1000} qualifying servers.`
);
if (!plan.requirements.length)
issues.push({
field: 'requirements',
message: 'Add at least one requirement to continue.'
});
for (const r of plan.requirements) {
if (!r.label?.trim()) issues.push({ field: `${r.id}-label`, message: 'Describe what this requirement checks.', requirementId: r.id });
const integral = ['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(r.kind);
check(`${r.id}-target`, r.target, r.kind === 'failure-budget' ? 0 : .000001, 1_000_000, integral, `Enter ${integral ? 'a whole number' : 'a number'} ${r.kind === 'failure-budget' ? 'of zero or more' : 'greater than zero'} for ${requirementLabels[r.kind].toLowerCase()}.`, r.id);
if (['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind)) check(`${r.id}-minPlayers`, r.minPlayers, 1, 1_000_000, true, 'Enter at least one player, using a whole number.', r.id);
if (!r.label?.trim())
issues.push({
field: `${r.id}-label`,
message: 'Describe what this requirement checks.',
requirementId: r.id
});
const integral = [
'round',
'matches',
'population',
'failure-budget',
'recovery',
'manual'
].includes(r.kind);
check(
`${r.id}-target`,
r.target,
r.kind === 'failure-budget' ? 0 : 0.000001,
1_000_000,
integral,
`Enter ${integral ? 'a whole number' : 'a number'} ${r.kind === 'failure-budget' ? 'of zero or more' : 'greater than zero'} for ${requirementLabels[r.kind].toLowerCase()}.`,
r.id
);
if (
[
'round',
'matches',
'populated-minutes',
'player-hours',
'sustained-population'
].includes(r.kind)
)
check(
`${r.id}-minPlayers`,
r.minPlayers,
1,
1_000_000,
true,
'Enter at least one player, using a whole number.',
r.id
);
if (r.kind === 'matches') {
check(`${r.id}-minRound`, r.minRound, 0, 1_000_000, true, 'Enter a whole round number; 0 allows any round.', r.id);
check(`${r.id}-minMinutes`, r.minMinutes, 0, 1_000_000, false, 'Enter a duration of zero or more minutes.', r.id);
check(
`${r.id}-minRound`,
r.minRound,
0,
1_000_000,
true,
'Enter a whole round number; 0 allows any round.',
r.id
);
check(
`${r.id}-minMinutes`,
r.minMinutes,
0,
1_000_000,
false,
'Enter a duration of zero or more minutes.',
r.id
);
}
}
return issues;
src/lib/beta-promotion.ts
+16 -4
View File
@@ -3,12 +3,24 @@ export interface BetaPromotionPreview {
token: string;
createdAt: string;
files: Array<{
id: number; name: string; kind: string; path: string; versionId: number;
version: number; release_version: string | null; sha256: string;
scope: string; servers: string[];
id: number;
name: string;
kind: string;
path: string;
versionId: number;
version: number;
release_version: string | null;
sha256: string;
scope: string;
servers: string[];
}>;
servers: Array<{ name: string; tested: boolean; game: string | null; map: string | null }>;
configWrites: Array<{ server: string; path: string; sha256: string | null; error: string | null }>;
configWrites: Array<{
server: string;
path: string;
sha256: string | null;
error: string | null;
}>;
blockers: string[];
restartPolicy: 'additional-when-empty';
}
src/lib/beta-revision.ts
+16 -4
View File
@@ -1,11 +1,23 @@
import type { BetaTestView } from './beta-plan.js';
export interface BetaRevisionFile {
script_id: number; name: string; kind: string; path: string;
version_id: number; version: number; release_version: string | null; sha256: string;
script_id: number;
name: string;
kind: string;
path: string;
version_id: number;
version: number;
release_version: string | null;
sha256: string;
}
export interface BetaRevision {
revision: number; started_at: string; notes: string | null;
revision: number;
started_at: string;
notes: string | null;
files: BetaRevisionFile[];
results: { state: string; test: BetaTestView | null; servers: Array<{ server: string; stage: string; error: string | null }> } | null;
results: {
state: string;
test: BetaTestView | null;
servers: Array<{ server: string; stage: string; error: string | null }>;
} | null;
}
src/lib/capacity.ts
+15 -4
View File
@@ -2,12 +2,19 @@ import type { CapacityCalibration, HostReadiness } from './hosts.js';
export type CapacityRange = '1h' | '24h' | '7d';
export type CapacityGame = 'all' | 't5' | 't6';
export interface CapacityQuery { range: CapacityRange; hostId: string | null; game: CapacityGame }
export interface CapacityQuery {
range: CapacityRange;
hostId: string | null;
game: CapacityGame;
}
export function parseCapacityQuery(params: URLSearchParams): CapacityQuery {
const range = params.get('range');
const game = params.get('game');
return { range: range === '1h' || range === '7d' ? range : '24h',
game: game === 't5' || game === 't6' ? game : 'all', hostId: params.get('host')?.trim().slice(0, 100) || null };
return {
range: range === '1h' || range === '7d' ? range : '24h',
game: game === 't5' || game === 't6' ? game : 'all',
hostId: params.get('host')?.trim().slice(0, 100) || null
};
}
export interface CapacityHistoryPoint {
@@ -60,4 +67,8 @@ export interface CapacityCurrent {
hostOptions: { id: string; name: string }[];
warnings: string[];
}
export interface CapacityData { query: CapacityQuery; current: CapacityCurrent; history: CapacityHistory }
export interface CapacityData {
query: CapacityQuery;
current: CapacityCurrent;
history: CapacityHistory;
}
src/lib/config-workbench.ts
+32 -15
View File
@@ -27,33 +27,45 @@ export interface ConfigInspection {
reason?: string;
}
export interface ConfigServer { name: string; game: string | null; map: string | null; }
export interface ConfigServer {
name: string;
game: string | null;
map: string | null;
}
/** The same lowercase `set` grammar that the overlay composer accepts. */
export function configLines(text: string) {
return text.split(/\r?\n/).flatMap((line, index) => {
const match = /^(\s*set\s+)(\S+)(\s+)(.*)$/.exec(line);
return match ? [{ index, key: match[2], value: match[4], prefix: match[1] + match[2] + match[3] }] : [];
return match
? [{ index, key: match[2], value: match[4], prefix: match[1] + match[2] + match[3] }]
: [];
});
}
export function changeConfigValue(text: string, index: number, value: string): string {
// Keep every separator, comment and unrelated line byte-for-byte intact.
const parts = text.split(/(\r?\n)/);
const entry = configLines(text).find(line => line.index === index);
const entry = configLines(text).find((line) => line.index === index);
if (entry) parts[index * 2] = entry.prefix + value.replace(/[\r\n]/g, '');
return parts.join('');
}
/** Add one assignment without changing existing commands or line endings. */
export function addConfigSetting(text: string, key: string, value: string): string {
key = key.trim(); value = value.trim();
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) throw new Error('Use a setting name containing letters, numbers, and underscores.');
if (!value || /[\r\n]/.test(value) || value.startsWith('//')) throw new Error('Enter a value on one line.');
if (configLines(text).some(entry => entry.key.toLowerCase() === key.toLowerCase())) throw new Error('This setting already exists. Find it above to edit its value.');
key = key.trim();
value = value.trim();
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key))
throw new Error('Use a setting name containing letters, numbers, and underscores.');
if (!value || /[\r\n]/.test(value) || value.startsWith('//'))
throw new Error('Enter a value on one line.');
if (configLines(text).some((entry) => entry.key.toLowerCase() === key.toLowerCase()))
throw new Error('This setting already exists. Find it above to edit its value.');
const newline = text.includes('\r\n') ? '\r\n' : '\n';
const formatted = /\s/.test(value) && !value.startsWith('"') ? JSON.stringify(value) : value;
return text + (text && !text.endsWith('\n') ? newline : '') + `set ${key} ${formatted}` + newline;
return (
text + (text && !text.endsWith('\n') ? newline : '') + `set ${key} ${formatted}` + newline
);
}
export function configWarnings(text: string, overlay: boolean): string[] {
@@ -61,15 +73,20 @@ export function configWarnings(text: string, overlay: boolean): string[] {
const seen = new Map<string, number>();
for (const entry of configLines(text)) {
const key = entry.key.toLowerCase();
if (seen.has(key)) warnings.push(`Line ${entry.index + 1}: ${entry.key} is also assigned on line ${seen.get(key)}.`);
if (seen.has(key))
warnings.push(
`Line ${entry.index + 1}: ${entry.key} is also assigned on line ${seen.get(key)}.`
);
seen.set(key, entry.index + 1);
if (!entry.value.trim() || entry.value.trim().startsWith('//')) warnings.push(`Line ${entry.index + 1}: ${entry.key} has no value.`);
if (!entry.value.trim() || entry.value.trim().startsWith('//'))
warnings.push(`Line ${entry.index + 1}: ${entry.key} has no value.`);
}
if (overlay) text.split(/\r?\n/).forEach((line, index) => {
if (line.trim() && !/^\s*\/\//.test(line) && !/^\s*set\s+\S+\s+/.test(line)) {
warnings.push(`Line ${index + 1} is ignored by overlays. Use set KEY VALUE.`);
}
});
if (overlay)
text.split(/\r?\n/).forEach((line, index) => {
if (line.trim() && !/^\s*\/\//.test(line) && !/^\s*set\s+\S+\s+/.test(line)) {
warnings.push(`Line ${index + 1} is ignored by overlays. Use set KEY VALUE.`);
}
});
if (/^\s*(?:(?:set|seta)\s+)?sv_maprotation\s+/im.test(text)) {
warnings.push('For recognized maps, the server’s selected map overrides sv_maprotation.');
}
src/lib/deployment-plan.ts
+41 -7
View File
@@ -55,9 +55,21 @@ export interface CollectionPreview {
mode: CollectionMode;
retainedServers: string[];
removals: { name: string; reason: string }[];
entries: { count: number; existing: number; missing: number; existingServers: string[];
coverage: { name: string; health: string; href: string }[]; recipe: LaunchRecipeSummary }[];
servers: { native?: import('./hosts.js').NativePlacement; entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[];
entries: {
count: number;
existing: number;
missing: number;
existingServers: string[];
coverage: { name: string; health: string; href: string }[];
recipe: LaunchRecipeSummary;
}[];
servers: {
native?: import('./hosts.js').NativePlacement;
entryIndex: number;
name: string | null;
port: number | null;
hasSavedKey: boolean;
}[];
blockers: string[];
warnings?: string[];
availability?: LaunchAvailability;
@@ -87,10 +99,32 @@ export interface CollectionDetail {
canCancel: boolean;
retainedServers: string[];
retainedServerDeletedAt?: Record<string, string>;
queue: { concurrency: number; active: { id: string; server: string; state: string; stage: string }[]; queued: number };
servers: { id: string; server: string; action: 'create' | 'destroy'; port: number | null; map_slug: string | null; state: string; stage: string;
started_at: string; finished_at: string | null; error: string | null; server_exists: number; server_deleted_at?: string | null; queuePosition: number | null;
cancellation: { cleanupId: string; state: string; stage: string; error: string | null } | null }[];
queue: {
concurrency: number;
active: { id: string; server: string; state: string; stage: string }[];
queued: number;
};
servers: {
id: string;
server: string;
action: 'create' | 'destroy';
port: number | null;
map_slug: string | null;
state: string;
stage: string;
started_at: string;
finished_at: string | null;
error: string | null;
server_exists: number;
server_deleted_at?: string | null;
queuePosition: number | null;
cancellation: {
cleanupId: string;
state: string;
stage: string;
error: string | null;
} | null;
}[];
}
export interface LaunchPreview {
src/lib/fleet-overview.ts
+77 -20
View File
@@ -9,7 +9,10 @@ import { mapTitle, serverTitle } from './ui/server-labels.js';
export interface FleetServer {
plutonium?: import('./plutonium.js').PlutoniumVersion;
host_id?: string | null;
host?: Pick<import('./hosts.js').Host, 'id' | 'name' | 'connectionState' | 'connected' | 'lastHeartbeat'> | null;
host?: Pick<
import('./hosts.js').Host,
'id' | 'name' | 'connectionState' | 'connected' | 'lastHeartbeat'
> | null;
container_state?: string | null;
container_checked_at?: string | null;
last_checked_at?: string | null;
@@ -56,49 +59,103 @@ export type FleetFilter = 'all' | 'occupied' | 'empty' | 'attention' | 'testing'
export type FleetSort = 'name' | 'players' | 'round' | 'duration';
export function freshServer(server: FleetServer, now: number): boolean {
return server.health === 'online' && !server.last_error && freshStatusTimestamp(server.last_seen_at, now);
return (
server.health === 'online' &&
!server.last_error &&
freshStatusTimestamp(server.last_seen_at, now)
);
}
export function fleetIdentity(server: FleetServer) {
const parsed = parseServerName(server.name);
const game = server.game ?? parsed?.game ?? '';
const mode = server.game_mode ?? parsed?.gameMode;
const selection = !mode || mode === 'zombie'
? zombieMapBySlug(game, server.map_slug ?? parsed?.mapSlug ?? '') : undefined;
const selection =
!mode || mode === 'zombie'
? zombieMapBySlug(game, server.map_slug ?? parsed?.mapSlug ?? '')
: undefined;
const map = server.last_map ?? server.map;
// A live map change must not keep the registered map's launch shortcut.
const matchingSelection = selection && (!map || selection.canonical === map) ? selection : undefined;
const matchingSelection =
selection && (!map || selection.canonical === map) ? selection : undefined;
return {
game,
title: serverTitle(server.name),
map: matchingSelection?.friendly ?? mapTitle(map),
group: `${game}:${matchingSelection?.slug ?? map ?? 'unknown'}`,
launch: matchingSelection && (game === 't5' || game === 't6')
? { game: game as ZombieGame, slug: matchingSelection.slug } : null
launch:
matchingSelection && (game === 't5' || game === 't6')
? { game: game as ZombieGame, slug: matchingSelection.slug }
: null
};
}
export function currentRound(server: FleetServer, report: GameTelemetry | undefined, now: number): number | null {
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
if (!report || (report.event !== 'round_change' && report.event !== 'heartbeat') || !freshServer(server, now) || !server.active_since || !server.game_started_at
|| server.game_ended_at || (server.last_players ?? 0) === 0) return null;
export function currentRound(
server: FleetServer,
report: GameTelemetry | undefined,
now: number
): number | null {
if (
server.telemetry &&
(!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))
)
report = server.telemetry;
if (
!report ||
(report.event !== 'round_change' && report.event !== 'heartbeat') ||
!freshServer(server, now) ||
!server.active_since ||
!server.game_started_at ||
server.game_ended_at ||
(server.last_players ?? 0) === 0
)
return null;
const received = Date.parse(report.receivedAt);
return freshTelemetry(report, now, server.telemetry_after) && received >= Date.parse(server.game_started_at)
&& received >= Date.parse(server.active_since) ? report.round : null;
return freshTelemetry(report, now, server.telemetry_after) &&
received >= Date.parse(server.game_started_at) &&
received >= Date.parse(server.active_since)
? report.round
: null;
}
export function matchesFleet(server: FleetServer, query: string, filter: FleetFilter, game: string,
pinned: readonly string[], operations: readonly FleetOperation[], now: number): boolean {
export function matchesFleet(
server: FleetServer,
query: string,
filter: FleetFilter,
game: string,
pinned: readonly string[],
operations: readonly FleetOperation[],
now: number
): boolean {
const identity = fleetIdentity(server);
if (game && identity.game !== game) return false;
const fresh = freshServer(server, now);
if (filter === 'occupied' && !(fresh && (server.last_players ?? 0) > 0)) return false;
if (filter === 'empty' && !(fresh && server.last_players === 0)) return false;
if (filter === 'attention' && !(server.health === 'offline' || server.health === 'stale'
|| (server.health === 'online' && !fresh) || operations.some(op => op.server === server.name && op.state === 'attention'))) return false;
if (
filter === 'attention' &&
!(
server.health === 'offline' ||
server.health === 'stale' ||
(server.health === 'online' && !fresh) ||
operations.some((op) => op.server === server.name && op.state === 'attention')
)
)
return false;
if (filter === 'testing' && !server.held.length) return false;
if (filter === 'pinned' && !pinned.includes(server.name)) return false;
const text = [server.name, identity.title, identity.map, identity.game,
...(server.roster ?? []).map(player => player.name)].join(' ').toLocaleLowerCase();
return query.trim().toLocaleLowerCase().split(/\s+/).every(term => text.includes(term));
const text = [
server.name,
identity.title,
identity.map,
identity.game,
...(server.roster ?? []).map((player) => player.name)
]
.join(' ')
.toLocaleLowerCase();
return query
.trim()
.toLocaleLowerCase()
.split(/\s+/)
.every((term) => text.includes(term));
}
src/lib/fleet-settings.ts
+186 -53
View File
@@ -33,7 +33,6 @@ export interface FleetSettings {
lifecycleConcurrency: number;
rebuildConcurrency: number;
preparationConcurrency: number;
}
export const defaultFleetSettings: FleetSettings = {
@@ -70,7 +69,7 @@ export const defaultFleetSettings: FleetSettings = {
destroyConcurrencyPerHost: 0,
lifecycleConcurrency: 0,
rebuildConcurrency: 0,
preparationConcurrency: 0,
preparationConcurrency: 0
};
export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
@@ -107,12 +106,12 @@ export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
destroyConcurrencyPerHost: 'Concurrent server removals per host',
lifecycleConcurrency: 'Combined lifecycle limit',
rebuildConcurrency: 'Concurrent rebuilds',
preparationConcurrency: 'Hosts preparing game files simultaneously',
preparationConcurrency: 'Hosts preparing game files simultaneously'
};
export function validateFleetSettings(input: unknown): FleetSettings {
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide fleet settings.');
if (!input || typeof input !== 'object' || Array.isArray(input))
throw new Error('Provide fleet settings.');
const values = input as Record<string, unknown>;
for (const key of Object.keys(values)) {
if (!Object.hasOwn(defaultFleetSettings, key)) throw new Error(`Unknown setting: ${key}`);
@@ -121,20 +120,68 @@ export function validateFleetSettings(input: unknown): FleetSettings {
const value = values[key];
const minimum = settingMinimum(key);
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) {
throw new Error(`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`);
throw new Error(
`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`
);
}
if (key.endsWith('Days') && value > 36500) {
throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`);
}
}
return Object.fromEntries(Object.keys(defaultFleetSettings).map(key => [key, values[key]])) as unknown as FleetSettings;
return Object.fromEntries(
Object.keys(defaultFleetSettings).map((key) => [key, values[key]])
) as unknown as FleetSettings;
}
export const additionalConcurrencyKeys: (keyof FleetSettings)[] = ["restartConcurrencyPerHost", "createConcurrencyPerHost", "destroyConcurrencyPerHost", "lifecycleConcurrency", "rebuildConcurrency", "preparationConcurrency"];
export const concurrencyKeys: (keyof FleetSettings)[] = ['restartConcurrency', 'createConcurrency', 'destroyConcurrency', ...additionalConcurrencyKeys];
export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays', 'provisioningKeepDays', 'actionKeepDays', 'failureKeepDays', 'maintenanceKeepDays', 'recoveryKeepDays', 'hostResourceKeepDays', 'instanceResourceKeepDays', 'metricsKeepDays', 'telemetryKeepDays', 'telemetryMaxPerServer', 'telemetryMaxTotal', 'matchKeepDays', 'betaKeepDays', 'fileContentKeepDays', 'payloadKeepDays', 'versionKeepDays', 'versionKeepCount', 'gameFileGraceDays', 'gameFileKeepCount', 'uploadKeepDays', 'auditKeepDays', 'historyKeepCount'];
export const additionalConcurrencyKeys: (keyof FleetSettings)[] = [
'restartConcurrencyPerHost',
'createConcurrencyPerHost',
'destroyConcurrencyPerHost',
'lifecycleConcurrency',
'rebuildConcurrency',
'preparationConcurrency'
];
export const concurrencyKeys: (keyof FleetSettings)[] = [
'restartConcurrency',
'createConcurrency',
'destroyConcurrency',
...additionalConcurrencyKeys
];
export const retentionKeys: (keyof FleetSettings)[] = [
'deployKeepDays',
'observationKeepDays',
'provisioningKeepDays',
'actionKeepDays',
'failureKeepDays',
'maintenanceKeepDays',
'recoveryKeepDays',
'hostResourceKeepDays',
'instanceResourceKeepDays',
'metricsKeepDays',
'telemetryKeepDays',
'telemetryMaxPerServer',
'telemetryMaxTotal',
'matchKeepDays',
'betaKeepDays',
'fileContentKeepDays',
'payloadKeepDays',
'versionKeepDays',
'versionKeepCount',
'gameFileGraceDays',
'gameFileKeepCount',
'uploadKeepDays',
'auditKeepDays',
'historyKeepCount'
];
export function settingMinimum(key: keyof FleetSettings): number {
return ['restartConcurrency','createConcurrency','destroyConcurrency','gameFileKeepCount'].includes(key) ? 1 : 0;
return [
'restartConcurrency',
'createConcurrency',
'destroyConcurrency',
'gameFileKeepCount'
].includes(key)
? 1
: 0;
}
export interface SettingsGroup {
@@ -146,34 +193,94 @@ export interface SettingsGroup {
/** Presentation groups do not change the saved schema or the section PATCH API. */
export const settingsGroups: SettingsGroup[] = [
{ id: 'concurrency', title: 'Concurrency', description: 'Limit simultaneous server operations and game-file preparation.', keys: concurrencyKeys },
{ id: 'operation-history', title: 'Operation history', description: 'Campaigns, server operations, maintenance, and routine checks.', keys: [
'deployKeepDays', 'provisioningKeepDays', 'actionKeepDays', 'observationKeepDays', 'failureKeepDays',
'maintenanceKeepDays', 'recoveryKeepDays', 'betaKeepDays', 'historyKeepCount'
] },
{ id: 'matches-players', title: 'Matches & players', description: 'Saved matches, player records, and leaderboard history.', keys: ['matchKeepDays'] },
{ id: 'telemetry-metrics', title: 'Telemetry & metrics', description: 'Game diagnostics, fleet activity, and host resource samples.', keys: [
'telemetryKeepDays', 'telemetryMaxPerServer', 'telemetryMaxTotal', 'metricsKeepDays', 'hostResourceKeepDays', 'instanceResourceKeepDays'
] },
{ id: 'files-versions', title: 'Files & versions', description: 'File snapshots, saved versions, game files, and abandoned uploads.', keys: [
'fileContentKeepDays', 'payloadKeepDays', 'versionKeepDays', 'versionKeepCount', 'gameFileGraceDays', 'gameFileKeepCount', 'uploadKeepDays'
] },
{ id: 'accounts-audit', title: 'Accounts & audit', description: 'Sign-in and account administration records.', keys: ['auditKeepDays'] }
{
id: 'concurrency',
title: 'Concurrency',
description: 'Limit simultaneous server operations and game-file preparation.',
keys: concurrencyKeys
},
{
id: 'operation-history',
title: 'Operation history',
description: 'Campaigns, server operations, maintenance, and routine checks.',
keys: [
'deployKeepDays',
'provisioningKeepDays',
'actionKeepDays',
'observationKeepDays',
'failureKeepDays',
'maintenanceKeepDays',
'recoveryKeepDays',
'betaKeepDays',
'historyKeepCount'
]
},
{
id: 'matches-players',
title: 'Matches & players',
description: 'Saved matches, player records, and leaderboard history.',
keys: ['matchKeepDays']
},
{
id: 'telemetry-metrics',
title: 'Telemetry & metrics',
description: 'Game diagnostics, fleet activity, and host resource samples.',
keys: [
'telemetryKeepDays',
'telemetryMaxPerServer',
'telemetryMaxTotal',
'metricsKeepDays',
'hostResourceKeepDays',
'instanceResourceKeepDays'
]
},
{
id: 'files-versions',
title: 'Files & versions',
description: 'File snapshots, saved versions, game files, and abandoned uploads.',
keys: [
'fileContentKeepDays',
'payloadKeepDays',
'versionKeepDays',
'versionKeepCount',
'gameFileGraceDays',
'gameFileKeepCount',
'uploadKeepDays'
]
},
{
id: 'accounts-audit',
title: 'Accounts & audit',
description: 'Sign-in and account administration records.',
keys: ['auditKeepDays']
}
];
/** Zero has a different meaning for age limits, count caps, and minimums. */
export function settingZeroMode(key: keyof FleetSettings): string | null {
if (settingMinimum(key) > 0) return null;
if (additionalConcurrencyKeys.includes(key) || key === 'telemetryMaxPerServer' || key === 'telemetryMaxTotal') return 'No limit';
if (
additionalConcurrencyKeys.includes(key) ||
key === 'telemetryMaxPerServer' ||
key === 'telemetryMaxTotal'
)
return 'No limit';
if (key === 'versionKeepDays') return 'No age expiry';
if (key === 'gameFileGraceDays') return 'No delay';
if (key === 'versionKeepCount' || key === 'historyKeepCount') return 'No minimum';
return key.endsWith('Days') ? 'Forever' : null;
}
export function settingUnit(key: keyof FleetSettings): 'days' | 'records' | 'versions' | 'operations' | 'hosts' {
export function settingUnit(
key: keyof FleetSettings
): 'days' | 'records' | 'versions' | 'operations' | 'hosts' {
if (key.endsWith('Days')) return 'days';
if (key === 'telemetryMaxPerServer' || key === 'telemetryMaxTotal' || key === 'historyKeepCount') return 'records';
if (
key === 'telemetryMaxPerServer' ||
key === 'telemetryMaxTotal' ||
key === 'historyKeepCount'
)
return 'records';
if (key === 'versionKeepCount' || key === 'gameFileKeepCount') return 'versions';
return key === 'preparationConcurrency' ? 'hosts' : 'operations';
}
@@ -188,38 +295,64 @@ export function settingDisplayValue(key: keyof FleetSettings, value: number): st
}
export const settingHelp: Partial<Record<keyof FleetSettings, string>> = {
provisioningKeepDays: 'Completed lifecycle history. Recovery, source workflows and cancellation dependencies remain protected.',
actionKeepDays: 'Completed agent actions. Referenced commands and retry evidence remain protected. Expired actions keep compact receipts to prevent duplicate execution.',
failureKeepDays: 'Explicitly resolved failed operations and campaigns, measured from resolution. Unresolved failures remain protected.',
maintenanceKeepDays: 'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.',
recoveryKeepDays: 'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.',
provisioningKeepDays:
'Completed lifecycle history. Recovery, source workflows and cancellation dependencies remain protected.',
actionKeepDays:
'Completed agent actions. Referenced commands and retry evidence remain protected. Expired actions keep compact receipts to prevent duplicate execution.',
failureKeepDays:
'Explicitly resolved failed operations and campaigns, measured from resolution. Unresolved failures remain protected.',
maintenanceKeepDays:
'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.',
recoveryKeepDays:
'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.',
hostResourceKeepDays: 'Host CPU and memory history.',
instanceResourceKeepDays: 'Minute resource samples. Capacity learning continues to use its independent 24-hour window.',
instanceResourceKeepDays:
'Minute resource samples. Capacity learning continues to use its independent 24-hour window.',
metricsKeepDays: 'Historical fleet occupancy and utilization samples.',
telemetryKeepDays: 'Raw received telemetry diagnostics. Independent of saved matches. Count limits also apply.',
telemetryKeepDays:
'Raw received telemetry diagnostics. Independent of saved matches. Count limits also apply.',
telemetryMaxPerServer: 'Count cap per server, in addition to the age limit.',
telemetryMaxTotal: 'Fleet-wide count cap for telemetry diagnostics, in addition to the age limit.',
matchKeepDays: 'Completed and interrupted matches and player details. Expiry changes leaderboards and player totals; no lifetime aggregates are preserved.',
betaKeepDays: 'Closed betas, revisions and test evidence, after linked campaigns and recovery work no longer need them.',
fileContentKeepDays: 'Expired campaign file bodies are removed while summaries remain. Rollback becomes unavailable for those campaigns. Active beta snapshots remain protected.',
payloadKeepDays: 'Expire file bodies in completed launch recipes, commands and receipts while keeping their history. Unfinished work and protected records keep their files.',
versionKeepDays: 'Unused script, plugin, configuration and baseline versions. With no age expiry, the version count policy may still remove older unused versions. Current and referenced versions are always protected.',
versionKeepCount: 'Keep at least this many recent versions per artifact. With no age expiry, a positive count removes older unused versions.',
gameFileGraceDays: 'Delay removal after a version becomes superseded. Latest files, referenced versions and offline host copies remain protected.',
gameFileKeepCount: 'Keep this many recent published versions per game, in addition to every referenced or protected version.',
uploadKeepDays: 'Remove incomplete browser uploads after this many days without activity. Active transfers and publication are protected.',
telemetryMaxTotal:
'Fleet-wide count cap for telemetry diagnostics, in addition to the age limit.',
matchKeepDays:
'Completed and interrupted matches and player details. Expiry changes leaderboards and player totals; no lifetime aggregates are preserved.',
betaKeepDays:
'Closed betas, revisions and test evidence, after linked campaigns and recovery work no longer need them.',
fileContentKeepDays:
'Expired campaign file bodies are removed while summaries remain. Rollback becomes unavailable for those campaigns. Active beta snapshots remain protected.',
payloadKeepDays:
'Expire file bodies in completed launch recipes, commands and receipts while keeping their history. Unfinished work and protected records keep their files.',
versionKeepDays:
'Unused script, plugin, configuration and baseline versions. With no age expiry, the version count policy may still remove older unused versions. Current and referenced versions are always protected.',
versionKeepCount:
'Keep at least this many recent versions per artifact. With no age expiry, a positive count removes older unused versions.',
gameFileGraceDays:
'Delay removal after a version becomes superseded. Latest files, referenced versions and offline host copies remain protected.',
gameFileKeepCount:
'Keep this many recent published versions per game, in addition to every referenced or protected version.',
uploadKeepDays:
'Remove incomplete browser uploads after this many days without activity. Active transfers and publication are protected.',
auditKeepDays: 'Recorded authentication and account-administration events.',
historyKeepCount: 'Keep at least this many recent records per history category. Does not apply to metrics, resource samples, telemetry count caps or game-file storage.',
historyKeepCount:
'Keep at least this many recent records per history category. Does not apply to metrics, resource samples, telemetry count caps or game-file storage.',
restartConcurrency: 'Fleet recovery slots stay occupied until container and game health are verified.',
createConcurrency: 'Fleet startup slots cover creation and rebuilds through health and selected-map verification.',
restartConcurrency:
'Fleet recovery slots stay occupied until container and game health are verified.',
createConcurrency:
'Fleet startup slots cover creation and rebuilds through health and selected-map verification.',
destroyConcurrency: 'Fleet removals hold capacity until teardown and cleanup finish.',
restartConcurrencyPerHost: 'Additional recovery limit on each host, including starts and rebuilds.',
restartConcurrencyPerHost:
'Additional recovery limit on each host, including starts and rebuilds.',
createConcurrencyPerHost: 'Additional startup limit on each host, including rebuilds.',
destroyConcurrencyPerHost: 'Additional removal limit on each host, including migration source removal.',
lifecycleConcurrency: 'Combined fleet limit for creates, starts, stops, restarts, rebuilds and removals. Each workflow counts once.',
destroyConcurrencyPerHost:
'Additional removal limit on each host, including migration source removal.',
lifecycleConcurrency:
'Combined fleet limit for creates, starts, stops, restarts, rebuilds and removals. Each workflow counts once.',
rebuildConcurrency: 'Additional fleet limit for rebuilds, including host drain migrations.',
preparationConcurrency: 'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.',
deployKeepDays: 'Eligible campaign records and file diffs. Pending, unresolved failed, held and recovery-linked campaigns are protected.',
observationKeepDays: 'Successful monitoring and file-read records. Failures and actions are retained.'
preparationConcurrency:
'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.',
deployKeepDays:
'Eligible campaign records and file diffs. Pending, unresolved failed, held and recovery-linked campaigns are protected.',
observationKeepDays:
'Successful monitoring and file-read records. Failures and actions are retained.'
};
src/lib/game-chat.ts
+86 -28
View File
@@ -45,12 +45,20 @@ function logSeconds(time: string): number {
}
function plainText(value: string): string {
return value.replace(/\^[0-9]/g, '').replace(/[\x00-\x1f\x7f]/g, '').trim();
return (
value
.replace(/\^[0-9]/g, '')
// eslint-disable-next-line no-control-regex -- Strip control characters from displayed chat text.
.replace(/[\x00-\x1f\x7f]/g, '')
.trim()
);
}
/** Preserve log order: elapsed timestamps can reset between matches. */
export function parseGameChat(lines: readonly string[]): ChatMessage[] {
return parseGameChatTimeline(lines).flatMap(entry => entry.type === 'message' ? [entry.message] : []);
return parseGameChatTimeline(lines).flatMap((entry) =>
entry.type === 'message' ? [entry.message] : []
);
}
/** Keep match boundaries alongside messages, including when the log clock resets. */
@@ -89,8 +97,16 @@ function indexedTimeline(lines: readonly string[]): { entry: ChatEntry; lineInde
if (!match) continue;
const message = plainText(match[5]);
if (!message) continue;
push({ type: 'message', message: { time: match[1], guid: match[2], slot: Number(match[3]),
player: plainText(match[4]) || 'Unknown player', message } });
push({
type: 'message',
message: {
time: match[1],
guid: match[2],
slot: Number(match[3]),
player: plainText(match[4]) || 'Unknown player',
message
}
});
}
return entries;
}
@@ -100,11 +116,14 @@ function indexedTimeline(lines: readonly string[]): { entry: ChatEntry; lineInde
* segment to the file's last write, then join observed gameplay at that time.
* Resets have unknown downtime, so earlier segments cannot share this anchor.
*/
export function enrichGameChatTimeline(lines: readonly string[], options: {
mtime: string | null;
matches: readonly ChatMatch[];
reports: readonly ChatRoundReport[];
}): ContextualChatEntry[] {
export function enrichGameChatTimeline(
lines: readonly string[],
options: {
mtime: string | null;
matches: readonly ChatMatch[];
reports: readonly ChatRoundReport[];
}
): ContextualChatEntry[] {
let segmentStart = 0;
let previousSeconds: number | null = null;
let lastLine = -1;
@@ -115,30 +134,43 @@ export function enrichGameChatTimeline(lines: readonly string[], options: {
const event = line.slice(timestamp?.[0].length ?? 0).trimStart();
const lifecycle = /^(?:InitGame|ExitLevel|ShutdownGame)(?:\s|:|$)/.test(event);
if (!timestamp) {
if (lifecycle) { segmentStart = index + 1; previousSeconds = null; }
if (lifecycle) {
segmentStart = index + 1;
previousSeconds = null;
}
continue;
}
const seconds = logSeconds(timestamp[1]);
if (previousSeconds !== null && (seconds < previousSeconds
|| (seconds === 0 && /^InitGame(?:\s|:|$)/.test(event)))) segmentStart = index;
if (
previousSeconds !== null &&
(seconds < previousSeconds || (seconds === 0 && /^InitGame(?:\s|:|$)/.test(event)))
)
segmentStart = index;
previousSeconds = seconds;
}
// An untimed final write cannot locate the preceding timed event in real time.
const finalTimestamp = lastLine < 0 ? null : logTimestamp(lines[lastLine]);
const writtenAt = options.mtime ? Date.parse(options.mtime) : NaN;
const offset = finalTimestamp ? writtenAt - logSeconds(finalTimestamp[1]) * 1000 : NaN;
const reports = options.reports.map(report => ({ ...report, at: Date.parse(report.receivedAt) }))
const reports = options.reports
.map((report) => ({ ...report, at: Date.parse(report.receivedAt) }))
.sort((a, b) => b.at - a.at);
const matches = options.matches.map(match => ({ ...match, start: Date.parse(match.startedAt) }))
const matches = options.matches
.map((match) => ({ ...match, start: Date.parse(match.startedAt) }))
.sort((a, b) => b.start - a.start)
.map((match, index, sorted) => {
const nextStart = sorted[index - 1]?.start ?? Infinity;
// Identical heartbeats are deduplicated by the leaderboard, but still
// establish that this match was running when the message was sent.
const latest = reports.find(report => report.at >= match.start && report.at < nextStart);
const latest = reports.find(
(report) => report.at >= match.start && report.at < nextStart
);
const lastSeen = Math.max(Date.parse(match.lastReceivedAt), latest?.at ?? -Infinity);
const end = match.endedAt ? Date.parse(match.endedAt)
: match.status === 'active' ? lastSeen + 120_000 : Date.parse(match.lastReceivedAt);
const end = match.endedAt
? Date.parse(match.endedAt)
: match.status === 'active'
? lastSeen + 120_000
: Date.parse(match.lastReceivedAt);
return { ...match, end: Math.min(end, nextStart) };
});
let logStartedAt: number | null = null;
@@ -147,8 +179,10 @@ export function enrichGameChatTimeline(lines: readonly string[], options: {
return indexedTimeline(lines).map(({ entry, lineIndex }) => {
const context: ChatContext = { realTime: null, matchElapsedMs: null, round: null };
const time = entry.type === 'message' ? entry.message.time : entry.time;
const at = time !== null && lineIndex >= segmentStart ? offset + logSeconds(time) * 1000 : NaN;
if (Number.isFinite(at) && Number.isFinite(new Date(at).getTime())) context.realTime = new Date(at).toISOString();
const at =
time !== null && lineIndex >= segmentStart ? offset + logSeconds(time) * 1000 : NaN;
if (Number.isFinite(at) && Number.isFinite(new Date(at).getTime()))
context.realTime = new Date(at).toISOString();
if (entry.type === 'boundary' && entry.event !== 'end') {
// The first event after an implicit reset can be several seconds into
// the new process. Its clock origin is the lower bound, not that event.
@@ -156,20 +190,44 @@ export function enrichGameChatTimeline(lines: readonly string[], options: {
logStartedAt = context.realTime ? (entry.event === 'reset' ? offset : at) : Infinity;
logEnded = false;
}
const match = context.realTime && !logEnded ? matches.find(match => at >= match.start && at <= match.end
// Allow the log's one-second precision at a start, but never cross a later InitGame.
&& (logStartedAt === null || match.start + 1000 >= logStartedAt
|| reports.some(report => report.event === 'game_start' && report.playerCount > 0
&& report.at >= match.start && report.at <= at && report.at + 1000 >= logStartedAt!))) : undefined;
const match =
context.realTime && !logEnded
? matches.find(
(match) =>
at >= match.start &&
at <= match.end &&
// Allow the log's one-second precision at a start, but never cross a later InitGame.
(logStartedAt === null ||
match.start + 1000 >= logStartedAt ||
reports.some(
(report) =>
report.event === 'game_start' &&
report.playerCount > 0 &&
report.at >= match.start &&
report.at <= at &&
report.at + 1000 >= logStartedAt!
))
)
: undefined;
if (match) {
const report = reports.find(report => report.at <= at && report.at >= match.start);
const report = reports.find((report) => report.at <= at && report.at >= match.start);
if (!report || (report.event !== 'game_end' && report.playerCount > 0)) {
// A repeated start updates the fleet timer even if the leaderboard
// deduplicates it. Use that same observed start for later messages.
const start = reports.find(report => report.event === 'game_start' && report.playerCount > 0
&& report.at >= match.start && report.at <= at)?.at ?? (match.startObserved ? match.start : null);
const start =
reports.find(
(report) =>
report.event === 'game_start' &&
report.playerCount > 0 &&
report.at >= match.start &&
report.at <= at
)?.at ?? (match.startObserved ? match.start : null);
if (start !== null) context.matchElapsedMs = at - start;
if (report && at - report.at <= 120_000 && (report.event === 'round_change' || report.event === 'heartbeat')) {
if (
report &&
at - report.at <= 120_000 &&
(report.event === 'round_change' || report.event === 'heartbeat')
) {
context.round = report.round;
}
}
src/lib/game-files.ts
+48 -16
View File
@@ -4,33 +4,65 @@ export const GAME_FILE_GAMES = [
{ id: 't6', name: 'Black Ops II' }
] as const;
export type GameLibraryState = 'importing' | 'ready' | 'failed';
export type LibraryCacheState = 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated';
export type LibraryCacheState =
'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated';
export interface GameFileEntry { path: string; size: number; sha256: string }
export interface GameFileEntry {
path: string;
size: number;
sha256: string;
}
export interface LibraryTransferProgress {
phase: 'downloading' | 'verifying'; startedAt: string;
elapsedSeconds: number; phaseElapsedSeconds: number;
bytesPerSecond: number | null; etaSeconds: number | null;
phase: 'downloading' | 'verifying';
startedAt: string;
elapsedSeconds: number;
phaseElapsedSeconds: number;
bytesPerSecond: number | null;
etaSeconds: number | null;
}
export interface HostLibraryCache {
hostId: string; libraryVersionId: string; status: LibraryCacheState;
bytesDownloaded: number; bytesTotal: number; manifestSha256: string | null;
error: string | null; updatedAt: string; operationId: string | null;
bytesVerified?: number; transfer?: LibraryTransferProgress;
hostId: string;
libraryVersionId: string;
status: LibraryCacheState;
bytesDownloaded: number;
bytesTotal: number;
manifestSha256: string | null;
error: string | null;
updatedAt: string;
operationId: string | null;
bytesVerified?: number;
transfer?: LibraryTransferProgress;
}
export interface GameLibraryVersion {
id: string; libraryId: string; game: GameLibraryGame; version: string;
description: string; state: GameLibraryState; sizeBytes: number;
fileCount: number; manifestSha256: string | null; files: GameFileEntry[];
createdAt: string; publishedAt: string | null; storageLocation: string;
sourceHostId: string | null; sourceVolume: string | null; error: string | null; operationId: string | null;
id: string;
libraryId: string;
game: GameLibraryGame;
version: string;
description: string;
state: GameLibraryState;
sizeBytes: number;
fileCount: number;
manifestSha256: string | null;
files: GameFileEntry[];
createdAt: string;
publishedAt: string | null;
storageLocation: string;
sourceHostId: string | null;
sourceVolume: string | null;
error: string | null;
operationId: string | null;
caches: HostLibraryCache[];
filesRemoved?: boolean;
filesRemoving?: boolean;
}
export interface GameLibrary {
id: string; name: string; game: GameLibraryGame; description: string;
createdAt: string; latestVersionId: string | null; versions: GameLibraryVersion[];
id: string;
name: string;
game: GameLibraryGame;
description: string;
createdAt: string;
latestVersionId: string | null;
versions: GameLibraryVersion[];
}
export function formatGameBytes(bytes: number): string {
src/lib/host-operation-labels.ts
+24 -10
View File
@@ -1,14 +1,28 @@
export const hostOperationTitles: Record<string, string> = {
'instance.create': 'Create server container', 'instance.rebuild': 'Rebuild server container',
'instance.start': 'Start server container', 'instance.restart': 'Restart server container',
'instance.stop': 'Stop server container', 'instance.remove': 'Destroy server container',
'instance.cancel-create': 'Cancel server creation', 'instance.inspect': 'Check server container',
'host.inspect': 'Check host', 'host.decommission': 'Remove host', 'runtime.pull': 'Prepare container runtime',
'library.download': 'Download game files', 'library.import': 'Import game files',
'plutonium.prepare': 'Prepare game update', 'files.stage': 'Stage server files',
'files.apply': 'Apply server files', 'files.read': 'Read server files', 'files.list': 'List server files',
'files.stat': 'Check server files', 'files.manifest': 'Check file versions', 'files.remove': 'Remove server files',
'rcon.status-batch': 'Check game status', 'rcon.execute': 'Game command', 'logs.read': 'Read server logs'
'instance.create': 'Create server container',
'instance.rebuild': 'Rebuild server container',
'instance.start': 'Start server container',
'instance.restart': 'Restart server container',
'instance.stop': 'Stop server container',
'instance.remove': 'Destroy server container',
'instance.cancel-create': 'Cancel server creation',
'instance.inspect': 'Check server container',
'host.inspect': 'Check host',
'host.decommission': 'Remove host',
'runtime.pull': 'Prepare container runtime',
'library.download': 'Download game files',
'library.import': 'Import game files',
'plutonium.prepare': 'Prepare game update',
'files.stage': 'Stage server files',
'files.apply': 'Apply server files',
'files.read': 'Read server files',
'files.list': 'List server files',
'files.stat': 'Check server files',
'files.manifest': 'Check file versions',
'files.remove': 'Remove server files',
'rcon.status-batch': 'Check game status',
'rcon.execute': 'Game command',
'logs.read': 'Read server logs'
};
export function hostOperationTitle(kind: string): string {
src/lib/hosts.ts
+204 -58
View File
@@ -1,31 +1,67 @@
import type { HostLibraryCache } from './game-files.js';
export type HostMode = 'active' | 'draining' | 'disabled';
export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled' | 'deleting';
export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
export type HostReadiness =
| 'preparing'
| 'needs-runtime'
| 'needs-game-files'
| 'ready'
| 'degraded'
| 'draining'
| 'disabled'
| 'deleting';
export type HostOperationState =
'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
export function hostReadinessLabel(readiness: HostReadiness): string {
return { preparing: 'Preparing', 'needs-runtime': 'Needs runtime', 'needs-game-files': 'Needs game files',
ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled', deleting: 'Deleting' }[readiness];
return {
preparing: 'Preparing',
'needs-runtime': 'Needs runtime',
'needs-game-files': 'Needs game files',
ready: 'Ready',
degraded: 'Degraded',
draining: 'Draining',
disabled: 'Disabled',
deleting: 'Deleting'
}[readiness];
}
export interface HostDeletion { allowed: boolean; blockers: string[] }
export interface HostDeletion {
allowed: boolean;
blockers: string[];
}
export interface HostDecommission {
operationId: string; stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed';
requestedAt: string; updatedAt: string; error: string | null; retryable: boolean;
operationId: string;
stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed';
requestedAt: string;
updatedAt: string;
error: string | null;
retryable: boolean;
}
export function hostDeletionMessage(host: Pick<Host, 'connected' | 'decommission'>): string {
if (!host.decommission) return '';
switch (host.decommission.stage) {
case 'queued': return host.connected ? 'Host cleanup is queued.' : 'Waiting for the host to reconnect before removing its files and agent.';
case 'running': return 'Removing host files, cached game libraries, and the agent installation.';
case 'cleaned': return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.';
case 'failed': return 'Cleanup needs attention. Complete removal has not been confirmed.';
case 'completed': return 'Host files and agent removal confirmed.';
case 'queued':
return host.connected
? 'Host cleanup is queued.'
: 'Waiting for the host to reconnect before removing its files and agent.';
case 'running':
return 'Removing host files, cached game libraries, and the agent installation.';
case 'cleaned':
return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.';
case 'failed':
return 'Cleanup needs attention. Complete removal has not been confirmed.';
case 'completed':
return 'Host files and agent removal confirmed.';
}
}
export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string }
export interface HostLibrary {
id: string;
state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated';
progress?: number;
error?: string;
}
export interface HostAgentUpdate {
enabled: boolean;
state: 'current' | 'checking' | 'deferred' | 'updating' | 'updated' | 'failed';
@@ -37,50 +73,93 @@ export interface HostAgentUpdate {
export interface HostAgentLane {
name: 'health-1' | 'health-2' | 'read' | 'write';
alive: boolean; busy: boolean; lastPollAt: string | null; pollAgeSeconds: number;
alive: boolean;
busy: boolean;
lastPollAt: string | null;
pollAgeSeconds: number;
}
export function hostAgentLaneLabel(name: HostAgentLane['name']): string {
return { 'health-1': 'Health check 1', 'health-2': 'Health check 2', read: 'File reads', write: 'Actions' }[name];
return {
'health-1': 'Health check 1',
'health-2': 'Health check 2',
read: 'File reads',
write: 'Actions'
}[name];
}
export function hostAgentLaneState(lane: HostAgentLane): 'stopped' | 'stalled' | 'busy' | 'starting' | 'polling' {
export function hostAgentLaneState(
lane: HostAgentLane
): 'stopped' | 'stalled' | 'busy' | 'starting' | 'polling' {
if (!lane.alive) return 'stopped';
// Reads and mutations can legitimately take minutes. Health work has a
// bounded budget; an old poll there indicates a blocked worker even if busy.
if (lane.pollAgeSeconds > 90 && (!lane.busy || lane.name.startsWith('health-'))) return 'stalled';
if (lane.pollAgeSeconds > 90 && (!lane.busy || lane.name.startsWith('health-')))
return 'stalled';
return lane.busy ? 'busy' : lane.lastPollAt === null ? 'starting' : 'polling';
}
export function hostAgentUpdatesEnabled(inventory: HostInventory): boolean {
return inventory.agentUpdate?.enabled === true && inventory.capabilities?.includes('agent.update') === true;
return (
inventory.agentUpdate?.enabled === true &&
inventory.capabilities?.includes('agent.update') === true
);
}
export function hostAgentUpdateLabel(inventory: HostInventory): string {
if (!hostAgentUpdatesEnabled(inventory)) return 'Setup required';
const update = inventory.agentUpdate!;
if (update.state === 'current' && !update.checkedAt) return 'Awaiting first check';
return { current: 'Current', checking: 'Checking', deferred: 'Deferred',
updating: 'Updating', updated: 'Updated', failed: 'Needs attention' }[update.state];
return {
current: 'Current',
checking: 'Checking',
deferred: 'Deferred',
updating: 'Updating',
updated: 'Updated',
failed: 'Needs attention'
}[update.state];
}
export function hostAgentUpdateMessage(inventory: HostInventory): string {
if (!hostAgentUpdatesEnabled(inventory)) return 'Run the current installer once on this host to enable automatic agent updates.';
if (!hostAgentUpdatesEnabled(inventory))
return 'Run the current installer once on this host to enable automatic agent updates.';
const update = inventory.agentUpdate!;
switch (update.state) {
case 'current': return update.checkedAt ? 'The last check found no newer compatible agent release.' : 'The host timer will check for a compatible agent release automatically.';
case 'checking': return 'The host is checking the controller release and verifying its download.';
case 'deferred': return 'The updater is waiting for the reported condition to clear. The host checks again automatically.';
case 'updating': return 'The host is replacing agent code and checking that the new process connects successfully.';
case 'updated': return 'The new agent connected successfully. Automatic checks continue.';
case 'failed': return 'The update did not complete successfully. Review the reported error before retrying setup.';
case 'current':
return update.checkedAt
? 'The last check found no newer compatible agent release.'
: 'The host timer will check for a compatible agent release automatically.';
case 'checking':
return 'The host is checking the controller release and verifying its download.';
case 'deferred':
return 'The updater is waiting for the reported condition to clear. The host checks again automatically.';
case 'updating':
return 'The host is replacing agent code and checking that the new process connects successfully.';
case 'updated':
return 'The new agent connected successfully. Automatic checks continue.';
case 'failed':
return 'The update did not complete successfully. Review the reported error before retrying setup.';
}
}
export interface HostInventory {
hostname?: string; agentVersion?: string; protocolVersion?: number; os?: string; architecture?: string;
cpu?: string; cpuCount?: number; cpuUsagePercent?: number; memoryTotal?: number; memoryAvailable?: number;
storageTotal?: number; storageAvailable?: number; dockerVersion?: string; dockerRunning?: boolean;
directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[];
currentOperation?: string | null; [key: string]: unknown;
hostname?: string;
agentVersion?: string;
protocolVersion?: number;
os?: string;
architecture?: string;
cpu?: string;
cpuCount?: number;
cpuUsagePercent?: number;
memoryTotal?: number;
memoryAvailable?: number;
storageTotal?: number;
storageAvailable?: number;
dockerVersion?: string;
dockerRunning?: boolean;
directoriesWritable?: boolean;
runtimeImages?: string[];
libraries?: HostLibrary[];
currentOperation?: string | null;
[key: string]: unknown;
capabilities?: string[];
plutoniumCache?: { revision: number; bytes: number };
agentUpdate?: HostAgentUpdate;
@@ -90,37 +169,82 @@ export interface HostInventory {
instanceStates?: InstanceState[];
}
export interface InstanceState {
instanceId: string; generation: number; state: string; healthy: boolean; observedAt: string;
instanceId: string;
generation: number;
state: string;
healthy: boolean;
observedAt: string;
}
export interface InstanceResources {
instanceId: string; generation: number; uptimeSeconds: number; cpuCores: number; memoryBytes: number;
instanceId: string;
generation: number;
uptimeSeconds: number;
cpuCores: number;
memoryBytes: number;
}
export interface ResourceAllowance {
cpuCores: number; memoryBytes: number; source: 'host' | 'fleet' | 'bootstrap';
samples: number; observedAt: string | null;
cpuCores: number;
memoryBytes: number;
source: 'host' | 'fleet' | 'bootstrap';
samples: number;
observedAt: string | null;
}
export interface CapacityCalibration {
running: ResourceAllowance;
startup: ResourceAllowance;
}
export interface CapacityCalibration { running: ResourceAllowance; startup: ResourceAllowance }
export interface CapacitySample {
hostId: string; hostName: string | null; instanceId: string; server: string | null;
generation: number; phase: 'startup' | 'running'; timestamp: string; cpuCores: number; memoryBytes: number;
hostId: string;
hostName: string | null;
instanceId: string;
server: string | null;
generation: number;
phase: 'startup' | 'running';
timestamp: string;
cpuCores: number;
memoryBytes: number;
}
export interface CapacitySamplePage {
samples: CapacitySample[]; total: number; page: number; pageSize: number;
samples: CapacitySample[];
total: number;
page: number;
pageSize: number;
}
export interface Host {
id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number;
bindAddress: string; publicAddress: string; minimumFreeBytes: number; createdAt: string;
lastHeartbeat: string | null; connected: boolean; connectionState: 'online' | 'offline';
readiness: HostReadiness; readinessReasons: string[]; inventory: HostInventory;
enrolled: boolean; credentialActive: boolean;
id: string;
name: string;
hostname: string;
mode: HostMode;
portMin: number;
portMax: number;
bindAddress: string;
publicAddress: string;
minimumFreeBytes: number;
createdAt: string;
lastHeartbeat: string | null;
connected: boolean;
connectionState: 'online' | 'offline';
readiness: HostReadiness;
readinessReasons: string[];
inventory: HostInventory;
enrolled: boolean;
credentialActive: boolean;
decommission: HostDecommission | null;
preparation?: HostPreparation;
enablingSince?: string | null;
}
export interface HostOperation {
id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null;
payload: Record<string, unknown>; state: HostOperationState; result: unknown; error: string | null;
createdAt: string; updatedAt: string;
id: string;
hostId: string;
kind: string;
instanceId: string | null;
generation: number | null;
payload: Record<string, unknown>;
state: HostOperationState;
result: unknown;
error: string | null;
createdAt: string;
updatedAt: string;
}
export interface HostOperationSummary extends Omit<HostOperation, 'payload' | 'result'> {
hostName: string | null;
@@ -130,11 +254,16 @@ export interface HostOperationSummary extends Omit<HostOperation, 'payload' | 'r
routine: boolean;
}
export type HostOperationFilter = 'important' | 'routine' | 'all';
export interface HostOperationScope { host?: string; server?: string; operation?: string }
export interface HostOperationScope {
host?: string;
server?: string;
operation?: string;
}
export function hostOperationScope(params: URLSearchParams): HostOperationScope {
const scope: HostOperationScope = {};
for (const key of ['host', 'server', 'operation'] as const) {
const value = params.get(key);
// eslint-disable-next-line no-control-regex -- Reject control characters in operation scope filters.
if (value && value.length <= 512 && !/[\x00-\x1f\x7f]/.test(value)) scope[key] = value;
}
return scope;
@@ -143,28 +272,45 @@ export function isHostOperationFilter(value: unknown): value is HostOperationFil
return typeof value === 'string' && ['important', 'routine', 'all'].includes(value);
}
export interface HostOperationHistoryPage {
operations: HostOperationSummary[]; page: number; pageSize: 25; total: number; totalPages: number; filter: HostOperationFilter;
operations: HostOperationSummary[];
page: number;
pageSize: 25;
total: number;
totalPages: number;
filter: HostOperationFilter;
}
export interface HostPreparationItem {
kind: 'runtime' | 'library' | 'plutonium'; label: string;
state: 'waiting' | 'preparing' | 'ready' | 'blocked'; detail: string;
operationId?: string; bytesDownloaded?: number; bytesTotal?: number;
kind: 'runtime' | 'library' | 'plutonium';
label: string;
state: 'waiting' | 'preparing' | 'ready' | 'blocked';
detail: string;
operationId?: string;
bytesDownloaded?: number;
bytesTotal?: number;
cache?: HostLibraryCache;
}
export interface HostPreparation {
state: HostPreparationItem['state']; items: HostPreparationItem[];
state: HostPreparationItem['state'];
items: HostPreparationItem[];
}
export interface NativeDeploymentSelection {
libraryIds: Partial<Record<'t5' | 't6', string>>; image: string;
bindAddress?: string; publicAddress?: string; publicPort?: number;
libraryIds: Partial<Record<'t5' | 't6', string>>;
image: string;
bindAddress?: string;
publicAddress?: string;
publicPort?: number;
}
/** Queued automatic launches have no assigned host or port until provisioning. */
export interface NativePlacement {
hostId?: string; libraryId: string; image?: string;
bindAddress?: string; publicAddress?: string; publicPort?: number;
hostId?: string;
libraryId: string;
image?: string;
bindAddress?: string;
publicAddress?: string;
publicPort?: number;
automatic?: boolean;
network?: { bindAddress?: string; publicAddress?: string; publicPort?: number };
}
src/lib/leaderboard-efficiency.ts
+52 -22
View File
@@ -5,20 +5,40 @@ export const EFFICIENCY_MIN_KILLS = 100;
export const EFFICIENCY_MIN_DOWNS = 5;
export const efficiencyMetrics = [
{ key: 'headshotRate', label: 'Headshots / kills', format: 'percent',
description: `Total headshots divided by total kills in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_KILLS} kills.` },
{ key: 'killsPerDown', label: 'Kills per down', format: 'number',
description: `Total kills divided by total downs in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_DOWNS} downs.` },
{ key: 'averageKills', label: 'Average kills', format: 'number',
description: `Total kills divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.` },
{ key: 'averageRounds', label: 'Average rounds', format: 'number',
description: `Total observed rounds divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.` },
{ key: 'completionRate', label: 'Completion rate', format: 'percent',
description: `Games with the player in the final report divided by games played. Only ended games count, including games the player left early. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games.` }
{
key: 'headshotRate',
label: 'Headshots / kills',
format: 'percent',
description: `Total headshots divided by total kills in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_KILLS} kills.`
},
{
key: 'killsPerDown',
label: 'Kills per down',
format: 'number',
description: `Total kills divided by total downs in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_DOWNS} downs.`
},
{
key: 'averageKills',
label: 'Average kills',
format: 'number',
description: `Total kills divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.`
},
{
key: 'averageRounds',
label: 'Average rounds',
format: 'number',
description: `Total observed rounds divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.`
},
{
key: 'completionRate',
label: 'Completion rate',
format: 'percent',
description: `Games with the player in the final report divided by games played. Only ended games count, including games the player left early. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games.`
}
] as const;
export type EfficiencyMetric = typeof efficiencyMetrics[number]['key'];
const metrics = new Set<string>(efficiencyMetrics.map(metric => metric.key));
export type EfficiencyMetric = (typeof efficiencyMetrics)[number]['key'];
const metrics = new Set<string>(efficiencyMetrics.map((metric) => metric.key));
export function isEfficiencyMetric(metric: string): metric is EfficiencyMetric {
return metrics.has(metric);
@@ -26,11 +46,16 @@ export function isEfficiencyMetric(metric: string): metric is EfficiencyMetric {
function sample(stats: PlayerStats, metric: EfficiencyMetric): [number, number] {
switch (metric) {
case 'headshotRate': return [stats.headshots, stats.kills];
case 'killsPerDown': return [stats.kills, stats.downs];
case 'averageKills': return [stats.kills, stats.gamesPlayed];
case 'averageRounds': return [stats.roundsSurvived, stats.gamesPlayed];
case 'completionRate': return [stats.gamesCompleted, stats.gamesPlayed];
case 'headshotRate':
return [stats.headshots, stats.kills];
case 'killsPerDown':
return [stats.kills, stats.downs];
case 'averageKills':
return [stats.kills, stats.gamesPlayed];
case 'averageRounds':
return [stats.roundsSurvived, stats.gamesPlayed];
case 'completionRate':
return [stats.gamesCompleted, stats.gamesPlayed];
}
}
@@ -60,11 +85,16 @@ const count = (value: number, singular: string, plural = `${singular}s`) =>
export function efficiencySample(stats: PlayerStats, metric: EfficiencyMetric): string {
switch (metric) {
case 'headshotRate': return `${count(stats.headshots, 'headshot')} / ${count(stats.kills, 'kill')}`;
case 'killsPerDown': return `${count(stats.kills, 'kill')} / ${count(stats.downs, 'down')}`;
case 'averageKills': return `${count(stats.kills, 'kill')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'averageRounds': return `${count(stats.roundsSurvived, 'observed round')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'completionRate': return `${count(stats.gamesCompleted, 'finish', 'finishes')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'headshotRate':
return `${count(stats.headshots, 'headshot')} / ${count(stats.kills, 'kill')}`;
case 'killsPerDown':
return `${count(stats.kills, 'kill')} / ${count(stats.downs, 'down')}`;
case 'averageKills':
return `${count(stats.kills, 'kill')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'averageRounds':
return `${count(stats.roundsSurvived, 'observed round')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'completionRate':
return `${count(stats.gamesCompleted, 'finish', 'finishes')} / ${count(stats.gamesPlayed, 'ended game')}`;
}
}
src/lib/leaderboard-navigation.ts
+6 -3
View File
@@ -12,10 +12,12 @@ export function leaderboardReturnHref(value: string | null | undefined): string
const params = new URLSearchParams();
for (const key of contextKeys) {
const entry = url.searchParams.get(key);
if (entry || key === 'map' && entry !== null) params.set(key, entry!);
if (entry || (key === 'map' && entry !== null)) params.set(key, entry!);
}
return `/leaderboard${params.size ? `?${params}` : ''}`;
} catch { return '/leaderboard'; }
} catch {
return '/leaderboard';
}
}
export function playerContextHref(guid: string, source: string): string {
@@ -31,7 +33,8 @@ export function playerContextHref(guid: string, source: string): string {
if (game === 't5' || game === 't6') params.set('game', game);
if (map !== null && map.length <= 1024) params.set('map', map);
if (players && /^[1-4]$/.test(players)) params.set('players', players);
if (metric && leaderboardMetrics.some(item => item.key === metric)) params.set('metric', metric);
if (metric && leaderboardMetrics.some((item) => item.key === metric))
params.set('metric', metric);
return `/leaderboard/players/${encodeURIComponent(guid)}?${params}`;
}
src/lib/leaderboard.ts
+2 -2
View File
@@ -14,8 +14,8 @@ export const playerStatMetrics = [
] as const;
export const leaderboardMetrics = [...playerStatMetrics, ...efficiencyMetrics] as const;
export type LeaderboardMetric = typeof leaderboardMetrics[number]['key'];
export type PlayerStatMetric = typeof playerStatMetrics[number]['key'];
export type LeaderboardMetric = (typeof leaderboardMetrics)[number]['key'];
export type PlayerStatMetric = (typeof playerStatMetrics)[number]['key'];
export type PlayerStats = Record<PlayerStatMetric, number>;
export type PersonalBestMetric = 'highestRound' | 'bestKills' | 'bestScore';
src/lib/maintenance.ts
+6 -1
View File
@@ -2,7 +2,12 @@ import type { WatchdogOverview } from './watchdog.js';
import type { PlutoniumStatus } from './plutonium.js';
export type MaintenancePolicy = 'health' | 'drift' | 'updates';
export interface MaintenancePolicies { paused: boolean; health: boolean; drift: boolean; updates: boolean }
export interface MaintenancePolicies {
paused: boolean;
health: boolean;
drift: boolean;
updates: boolean;
}
export interface MaintenanceOverview {
policies: MaintenancePolicies;
recovery: WatchdogOverview;
src/lib/map-aliases.ts
+71 -24
View File
@@ -36,14 +36,54 @@ export const T5_ZOMBIE_MAPS: readonly T5MapEntry[] = Object.freeze([
// Rulesets from xerxes-at/T6ServerConfigs, also seeded by Plutainer.
// Several selections share an engine map; slug preserves the selected variant.
export const T6_ZOMBIE_MAPS: readonly ZombieMapEntry[] = Object.freeze([
{ friendly: 'TranZit', slug: 'transit', canonical: 'zm_transit', gameSettings: 'zm_classic_transit.cfg' },
{ friendly: 'Town Survival', slug: 'town', canonical: 'zm_transit', gameSettings: 'zm_standard_town.cfg' },
{ friendly: 'Farm Survival', slug: 'farm', canonical: 'zm_transit', gameSettings: 'zm_standard_farm.cfg' },
{ friendly: 'Bus Depot Survival', slug: 'busdepot', canonical: 'zm_transit', gameSettings: 'zm_standard_transit.cfg' },
{ friendly: 'Nuketown', slug: 'nuked', canonical: 'zm_nuked', gameSettings: 'zm_standard_nuked.cfg' },
{ friendly: 'Die Rise', slug: 'highrise', canonical: 'zm_highrise', gameSettings: 'zm_classic_rooftop.cfg' },
{ friendly: 'Mob of the Dead', slug: 'prison', canonical: 'zm_prison', gameSettings: 'zm_classic_prison.cfg' },
{ friendly: 'Buried', slug: 'buried', canonical: 'zm_buried', gameSettings: 'zm_classic_processing.cfg' },
{
friendly: 'TranZit',
slug: 'transit',
canonical: 'zm_transit',
gameSettings: 'zm_classic_transit.cfg'
},
{
friendly: 'Town Survival',
slug: 'town',
canonical: 'zm_transit',
gameSettings: 'zm_standard_town.cfg'
},
{
friendly: 'Farm Survival',
slug: 'farm',
canonical: 'zm_transit',
gameSettings: 'zm_standard_farm.cfg'
},
{
friendly: 'Bus Depot Survival',
slug: 'busdepot',
canonical: 'zm_transit',
gameSettings: 'zm_standard_transit.cfg'
},
{
friendly: 'Nuketown',
slug: 'nuked',
canonical: 'zm_nuked',
gameSettings: 'zm_standard_nuked.cfg'
},
{
friendly: 'Die Rise',
slug: 'highrise',
canonical: 'zm_highrise',
gameSettings: 'zm_classic_rooftop.cfg'
},
{
friendly: 'Mob of the Dead',
slug: 'prison',
canonical: 'zm_prison',
gameSettings: 'zm_classic_prison.cfg'
},
{
friendly: 'Buried',
slug: 'buried',
canonical: 'zm_buried',
gameSettings: 'zm_classic_processing.cfg'
},
{ friendly: 'Origins', slug: 'tomb', canonical: 'zm_tomb', gameSettings: 'zm_classic_tomb.cfg' }
]);
@@ -55,23 +95,29 @@ export function zombieMapBySlug(game: string, slug: string): ZombieMapEntry | un
return zombieMaps(game).find((entry) => entry.slug === slug.toLowerCase());
}
export function selectedZombieMap(game: string | null, map: string | null, slug?: string | null): ZombieMapEntry | undefined {
export function selectedZombieMap(
game: string | null,
map: string | null,
slug?: string | null
): ZombieMapEntry | undefined {
const maps = zombieMaps(game);
// T5 maps are unambiguous, including legacy friendly slugs. T6 variants
// require the stored selection; never fall back from an unknown variant.
return maps.find((entry) => entry.canonical === map && (
game !== 't6' || !slug || entry.slug === slug
|| entry.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === slug
));
return maps.find(
(entry) =>
entry.canonical === map &&
(game !== 't6' ||
!slug ||
entry.slug === slug ||
entry.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === slug)
);
}
export function mapRotation(entry: ZombieMapEntry): string {
return `${entry.gameSettings ? `execgts ${entry.gameSettings} ` : ''}map ${entry.canonical}`;
}
const BY_SLUG: ReadonlyMap<string, T5MapEntry> = new Map(
T5_ZOMBIE_MAPS.map((m) => [m.slug, m])
);
const BY_SLUG: ReadonlyMap<string, T5MapEntry> = new Map(T5_ZOMBIE_MAPS.map((m) => [m.slug, m]));
export function t5ZombieMapBySlug(slug: string): T5MapEntry | undefined {
return BY_SLUG.get(slug.toLowerCase());
@@ -80,10 +126,10 @@ export function t5ZombieMapBySlug(slug: string): T5MapEntry | undefined {
export type GameMode = 'zombie' | 'mp' | 'sp';
export interface ParsedServerName {
game: string; // 't5'
gameMode: GameMode | null; // 'zombie' — null when the second segment isn't a known mode
mapSlug: string; // 'cosmodrome' — joined without separators for lookup
mapCanonical: string | null;// 'zombie_cosmodrome' — null when slug not in catalog
game: string; // 't5'
gameMode: GameMode | null; // 'zombie' — null when the second segment isn't a known mode
mapSlug: string; // 'cosmodrome' — joined without separators for lookup
mapCanonical: string | null; // 'zombie_cosmodrome' — null when slug not in catalog
lobby: number | null;
}
@@ -122,10 +168,11 @@ export function parseServerName(name: string): ParsedServerName | null {
const mapSlug = slugParts.join('').toLowerCase();
const catalog = zombieMaps(game.toLowerCase());
const entry = gameMode === null || gameMode === 'zombie'
? catalog.find((m) => m.slug === mapSlug)
?? catalog.find((m) => m.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === mapSlug)
: undefined;
const entry =
gameMode === null || gameMode === 'zombie'
? (catalog.find((m) => m.slug === mapSlug) ??
catalog.find((m) => m.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === mapSlug))
: undefined;
const mapCanonical = entry?.canonical ?? null;
return { game, gameMode, mapSlug, mapCanonical, lobby };
}
src/lib/match-history-query.ts
+47 -11
View File
@@ -1,12 +1,26 @@
export const matchPageSize = 25;
export const matchSorts = ['started', 'round', 'duration', 'players'] as const;
export type MatchSort = typeof matchSorts[number];
export type MatchSort = (typeof matchSorts)[number];
export interface MatchHistoryQuery {
game: string; map: string; q: string; days: number;
game: string;
map: string;
q: string;
days: number;
status: '' | 'active' | 'completed' | 'interrupted';
sort: MatchSort; direction: 'asc' | 'desc'; page: number;
sort: MatchSort;
direction: 'asc' | 'desc';
page: number;
}
export const matchQueryKeys = ['game', 'map', 'q', 'days', 'status', 'sort', 'direction', 'page'] as const;
export const matchQueryKeys = [
'game',
'map',
'q',
'days',
'status',
'sort',
'direction',
'page'
] as const;
export function readMatchQuery(params: URLSearchParams): MatchHistoryQuery {
const game = params.get('game') ?? '';
const map = params.get('map') ?? '';
@@ -16,11 +30,29 @@ export function readMatchQuery(params: URLSearchParams): MatchHistoryQuery {
const sort = params.get('sort') || 'started';
const direction = params.get('direction') || 'desc';
const page = Number(params.get('page') || 1);
if (!['', 't5', 't6'].includes(game) || map.length > 1024 || q.length > 100
|| ![0, 7, 30, 90].includes(days) || !['', 'active', 'completed', 'interrupted'].includes(status)
|| !matchSorts.includes(sort as MatchSort) || !['asc', 'desc'].includes(direction)
|| !Number.isSafeInteger(page) || page < 1 || page > 1000000) throw new Error('Invalid match history filters.');
return { game, map, q, days, status: status as MatchHistoryQuery['status'], sort: sort as MatchSort, direction: direction as 'asc' | 'desc', page };
if (
!['', 't5', 't6'].includes(game) ||
map.length > 1024 ||
q.length > 100 ||
![0, 7, 30, 90].includes(days) ||
!['', 'active', 'completed', 'interrupted'].includes(status) ||
!matchSorts.includes(sort as MatchSort) ||
!['asc', 'desc'].includes(direction) ||
!Number.isSafeInteger(page) ||
page < 1 ||
page > 1000000
)
throw new Error('Invalid match history filters.');
return {
game,
map,
q,
days,
status: status as MatchHistoryQuery['status'],
sort: sort as MatchSort,
direction: direction as 'asc' | 'desc',
page
};
}
/** Namespaced filters also work inside an instance's Matches tab. */
export function matchApiParams(url: URL): URLSearchParams {
@@ -31,13 +63,17 @@ export function matchApiParams(url: URL): URLSearchParams {
}
return params;
}
export function matchHistoryHref(url: URL, changes: Partial<Record<keyof MatchHistoryQuery, string>>): string {
export function matchHistoryHref(
url: URL,
changes: Partial<Record<keyof MatchHistoryQuery, string>>
): string {
const next = new URL(url);
next.searchParams.delete('matches'); // Retire the former growing-list limit.
next.searchParams.delete('matchPage');
for (const [key, value] of Object.entries(changes)) {
const name = `match${key[0].toUpperCase()}${key.slice(1)}`;
if (value) next.searchParams.set(name, value); else next.searchParams.delete(name);
if (value) next.searchParams.set(name, value);
else next.searchParams.delete(name);
}
return next.pathname + next.search;
}
src/lib/metrics-insights.ts
+91 -47
View File
@@ -23,51 +23,95 @@ export interface MetricRecommendation {
/** Apply the same visible rules to all selected maps, including sparse observations. */
export function metricRecommendations(maps: MapMetrics[]): MetricRecommendation[] {
return maps.map(map => {
const utilization = map.observed > 0 ? map.active / map.observed : null;
const pressure = map.capacityObserved > 0 ? map.full / map.capacityObserved : null;
const knownShare = map.observed + map.unknown > 0 ? map.observed / (map.observed + map.unknown) : 0;
const base = {
game: map.game,
map: map.map,
evidence: `${metricDuration(map.active)} with players / ${metricDuration(map.observed)} known server time (${metricPercent(map.active, map.observed)}). ` +
(pressure === null ? 'Capacity unavailable.' : `All lobbies full ${metricDuration(map.full)} / ${metricDuration(map.capacityObserved)} known map time (${metricPercent(map.full, map.capacityObserved)}).`),
tentative: map.observed < HOSTING_RULES.minimumEvidenceHours || pressure === null || map.capacityObserved < HOSTING_RULES.minimumEvidenceHours || knownShare < 0.8
};
if (utilization === null) return {
...base, kind: 'capacity' as const, title: map.unknown > 0 ? 'Check occupancy reporting' : 'Let observations accumulate',
reason: map.unknown > 0
? 'Captured status is unknown. Check a server’s telemetry or RCON diagnostics before choosing a hosting change.'
: 'This map has no elapsed observations in the selected window. Wait for another status check or try a wider range.',
outcome: 'Watch for known server time in the next report.', tentative: true
};
if (pressure !== null && pressure >= HOSTING_RULES.pressureExpansion) return {
...base, kind: 'expand' as const, title: 'Trial one additional lobby',
reason: 'Recorded lobbies were all full for at least 20% of known capacity time. Try one extra lobby during busy hours.',
outcome: 'Compare all-full time and use of the additional lobby over the next comparable period.'
};
if (pressure === null && utilization >= HOSTING_RULES.occupancyExpansion) return {
...base, kind: 'expand' as const, title: 'Trial an extra lobby during busy hours',
reason: 'Players were present for at least 80% of known server time. Capacity is unknown, so this experiment uses occupancy as a proxy, not evidence of saturation.',
outcome: 'Check whether players use the extra lobby, and measure capacity if the server reports it.', tentative: true
};
if (utilization <= HOSTING_RULES.idleUtilization && (pressure === null || pressure < HOSTING_RULES.quietPressure)) return {
...base, kind: 'consolidate' as const, title: 'Trial less idle hosting',
reason: 'Players were present for at most 10% of known time, with less than 1% measured pressure or unavailable capacity. Review one idle lobby or a shorter hosting window; preserve access to this map.',
outcome: 'Watch for increased all-full time or loss of player-hours, and restore the lineup if the trial hurts availability.'
};
if (pressure === null) return {
...base, kind: 'capacity' as const, title: 'Check capacity during busy hours',
reason: 'Occupancy is recorded, but player limits are unavailable. Check capacity reporting while keeping the current lineup.',
outcome: 'Look for known-capacity time and all-full periods before the next hosting trial.'
};
return {
...base, kind: 'watch' as const, title: 'Keep the lineup and watch peaks',
reason: 'These observations do not trigger the expansion or idle-hosting rules. Keep the current lineup for another comparison period.',
outcome: 'Watch whether occupied time and measured capacity pressure change.'
};
}).sort((a, b) => {
const priority = { expand: 0, consolidate: 1, capacity: 2, watch: 3 };
return priority[a.kind] - priority[b.kind] || a.map.localeCompare(b.map) || a.game.localeCompare(b.game);
});
return maps
.map((map) => {
const utilization = map.observed > 0 ? map.active / map.observed : null;
const pressure = map.capacityObserved > 0 ? map.full / map.capacityObserved : null;
const knownShare =
map.observed + map.unknown > 0 ? map.observed / (map.observed + map.unknown) : 0;
const base = {
game: map.game,
map: map.map,
evidence:
`${metricDuration(map.active)} with players / ${metricDuration(map.observed)} known server time (${metricPercent(map.active, map.observed)}). ` +
(pressure === null
? 'Capacity unavailable.'
: `All lobbies full ${metricDuration(map.full)} / ${metricDuration(map.capacityObserved)} known map time (${metricPercent(map.full, map.capacityObserved)}).`),
tentative:
map.observed < HOSTING_RULES.minimumEvidenceHours ||
pressure === null ||
map.capacityObserved < HOSTING_RULES.minimumEvidenceHours ||
knownShare < 0.8
};
if (utilization === null)
return {
...base,
kind: 'capacity' as const,
title:
map.unknown > 0
? 'Check occupancy reporting'
: 'Let observations accumulate',
reason:
map.unknown > 0
? 'Captured status is unknown. Check a server’s telemetry or RCON diagnostics before choosing a hosting change.'
: 'This map has no elapsed observations in the selected window. Wait for another status check or try a wider range.',
outcome: 'Watch for known server time in the next report.',
tentative: true
};
if (pressure !== null && pressure >= HOSTING_RULES.pressureExpansion)
return {
...base,
kind: 'expand' as const,
title: 'Trial one additional lobby',
reason: 'Recorded lobbies were all full for at least 20% of known capacity time. Try one extra lobby during busy hours.',
outcome:
'Compare all-full time and use of the additional lobby over the next comparable period.'
};
if (pressure === null && utilization >= HOSTING_RULES.occupancyExpansion)
return {
...base,
kind: 'expand' as const,
title: 'Trial an extra lobby during busy hours',
reason: 'Players were present for at least 80% of known server time. Capacity is unknown, so this experiment uses occupancy as a proxy, not evidence of saturation.',
outcome:
'Check whether players use the extra lobby, and measure capacity if the server reports it.',
tentative: true
};
if (
utilization <= HOSTING_RULES.idleUtilization &&
(pressure === null || pressure < HOSTING_RULES.quietPressure)
)
return {
...base,
kind: 'consolidate' as const,
title: 'Trial less idle hosting',
reason: 'Players were present for at most 10% of known time, with less than 1% measured pressure or unavailable capacity. Review one idle lobby or a shorter hosting window; preserve access to this map.',
outcome:
'Watch for increased all-full time or loss of player-hours, and restore the lineup if the trial hurts availability.'
};
if (pressure === null)
return {
...base,
kind: 'capacity' as const,
title: 'Check capacity during busy hours',
reason: 'Occupancy is recorded, but player limits are unavailable. Check capacity reporting while keeping the current lineup.',
outcome:
'Look for known-capacity time and all-full periods before the next hosting trial.'
};
return {
...base,
kind: 'watch' as const,
title: 'Keep the lineup and watch peaks',
reason: 'These observations do not trigger the expansion or idle-hosting rules. Keep the current lineup for another comparison period.',
outcome: 'Watch whether occupied time and measured capacity pressure change.'
};
})
.sort((a, b) => {
const priority = { expand: 0, consolidate: 1, capacity: 2, watch: 3 };
return (
priority[a.kind] - priority[b.kind] ||
a.map.localeCompare(b.map) ||
a.game.localeCompare(b.game)
);
});
}
src/lib/metrics.ts
+15 -4
View File
@@ -22,11 +22,22 @@ export function parseMetricsQuery(params: URLSearchParams): MetricsQuery {
const requestedGame = params.get('game')?.trim() ?? '';
const game = /^[a-z0-9_-]{1,32}$/.test(requestedGame) ? requestedGame : null;
const requestedMap = params.get('map');
const map = game && requestedMap && requestedMap.trim() && requestedMap.length <= 200 && !/[\u0000-\u001f\u007f]/.test(requestedMap)
? requestedMap : null;
const map =
game &&
requestedMap &&
requestedMap.trim() &&
requestedMap.length <= 200 &&
// eslint-disable-next-line no-control-regex -- Reject control characters in requested map names.
!/[\u0000-\u001f\u007f]/.test(requestedMap)
? requestedMap
: null;
const requestedZone = params.get('timeZone');
return { days: parseMetricsDays(params.get('days')), game, map,
timeZone: validTimeZone(requestedZone) ? requestedZone : DEFAULT_TIME_ZONE };
return {
days: parseMetricsDays(params.get('days')),
game,
map,
timeZone: validTimeZone(requestedZone) ? requestedZone : DEFAULT_TIME_ZONE
};
}
/** All durations are hours. Observed is known server time, not collection coverage. */
src/lib/navigation-context.ts
+38 -6
View File
@@ -1,14 +1,25 @@
const origin = 'https://electric-cherry.invalid';
const pagePath = /^\/(?:$|activity|operations|autoscaling|account|capacity|metrics|server-keys|settings(?:\/users)?|servers\/[^/]+|hosts(?:\/[^/]+)?|collections(?:\/launches\/[^/]+)?|(?:scripts|configs|plugins|betas|campaigns|maintenance|provisioning|matches)(?:\/[^/]+)?|game-files|leaderboard(?:\/players\/[^/]+(?:\/[^/]+)?)?)$/;
const pagePath =
/^\/(?:$|activity|operations|autoscaling|account|capacity|metrics|server-keys|settings(?:\/users)?|servers\/[^/]+|hosts(?:\/[^/]+)?|collections(?:\/launches\/[^/]+)?|(?:scripts|configs|plugins|betas|campaigns|maintenance|provisioning|matches)(?:\/[^/]+)?|game-files|leaderboard(?:\/players\/[^/]+(?:\/[^/]+)?)?)$/;
/** Only return to application pages, never an external URL or a mutation/API endpoint. */
export function safeReturnHref(value: string | null | undefined, fallback: string): string {
if (!value || value.length > 12000 || !value.startsWith('/') || value.startsWith('//') || /[\\\x00-\x1f\x7f]/.test(value)) return fallback;
if (
!value ||
value.length > 12000 ||
!value.startsWith('/') ||
value.startsWith('//') ||
// eslint-disable-next-line no-control-regex -- Reject control characters in redirect paths.
/[\\\x00-\x1f\x7f]/.test(value)
)
return fallback;
try {
const url = new URL(value, origin);
if (url.origin !== origin || !pagePath.test(url.pathname)) return fallback;
return url.pathname + url.search + url.hash;
} catch { return fallback; }
} catch {
return fallback;
}
}
export function contextHref(destination: string, source: URL | string): string {
@@ -25,11 +36,32 @@ export function returnLabel(href: string): string {
if (path.startsWith('/servers/')) return 'Back to server';
if (path.startsWith('/leaderboard/players/')) return 'Back to player';
if (path.startsWith('/collections/launches/')) return 'Back to collection launch';
if (path === '/collections' && url.searchParams.get('view') === 'active') return 'Back to active launches';
if (path === '/collections' && url.searchParams.get('view') === 'history') return 'Back to launch history';
if (path === '/collections' && url.searchParams.get('view') === 'active')
return 'Back to active launches';
if (path === '/collections' && url.searchParams.get('view') === 'history')
return 'Back to launch history';
if (path.startsWith('/matches/')) return 'Back to match';
if (path === '/hosts/operations') return 'Back to host operation history';
if (path.startsWith('/hosts/')) return 'Back to host';
const names: Record<string, string> = { '': 'Servers', activity: 'Activity', operations: 'Activity', autoscaling: 'Autoscaling', collections: 'Collections', hosts: 'Hosts', capacity: 'Capacity', scripts: 'Scripts', configs: 'Configs', plugins: 'Plugins', 'game-files': 'Game Files', betas: 'Betas', campaigns: 'Campaigns', maintenance: 'Maintenance', provisioning: 'Server queues', leaderboard: 'Leaderboard', matches: 'Match History', metrics: 'Metrics' };
const names: Record<string, string> = {
'': 'Servers',
activity: 'Activity',
operations: 'Activity',
autoscaling: 'Autoscaling',
collections: 'Collections',
hosts: 'Hosts',
capacity: 'Capacity',
scripts: 'Scripts',
configs: 'Configs',
plugins: 'Plugins',
'game-files': 'Game Files',
betas: 'Betas',
campaigns: 'Campaigns',
maintenance: 'Maintenance',
provisioning: 'Server queues',
leaderboard: 'Leaderboard',
matches: 'Match History',
metrics: 'Metrics'
};
return `Back to ${names[path.split('/')[1]] ?? 'previous page'}`;
}
src/lib/navigation.ts
+99 -35
View File
@@ -1,52 +1,116 @@
import type { UserRole } from './auth.js';
export interface NavigationItem { label: string; href: string; shortcut?: string; matches?: (path: string) => boolean }
export interface NavigationGroup { id: string; label: string; admin?: boolean; items: NavigationItem[] }
export interface NavigationItem {
label: string;
href: string;
shortcut?: string;
matches?: (path: string) => boolean;
}
export interface NavigationGroup {
id: string;
label: string;
admin?: boolean;
items: NavigationItem[];
}
export const navigationGroups: NavigationGroup[] = [
{ id: 'fleet', label: 'Fleet', items: [
{ label: 'Servers', href: '/', shortcut: 'f', matches: path => path === '/' || path.startsWith('/servers/') },
{ label: 'Hosts', href: '/hosts', shortcut: 'o', matches: path => path === '/hosts' || /^\/hosts\/(?!operations(?:\/|$))/.test(path) },
{ label: 'Capacity', href: '/capacity' },
{ label: 'Autoscaling', href: '/autoscaling' },
{ label: 'Collections', href: '/collections' }
] },
{ id: 'releases', label: 'Releases', items: [
{ label: 'Scripts', href: '/scripts', shortcut: 's' },
{ label: 'Configs', href: '/configs', shortcut: 'c' },
{ label: 'Plugins', href: '/plugins', shortcut: 'p' },
{ label: 'Game files', href: '/game-files' },
{ label: 'Betas', href: '/betas', shortcut: 'b' },
{ label: 'Campaigns', href: '/campaigns', shortcut: 'a' }
] },
{ id: 'activity', label: 'Activity', items: [
{ label: 'Activity log', href: '/activity', shortcut: 'd', matches: path => path === '/activity' || path === '/operations' || path === '/hosts/operations' },
{ label: 'Provisioning', href: '/provisioning', matches: path => /^\/(provisioning|deployments)(\/|$)/.test(path) },
{ label: 'Maintenance', href: '/maintenance', shortcut: 'w', matches: path => /^\/(maintenance|watchdog)(\/|$)/.test(path) }
] },
{ id: 'players', label: 'Players', items: [
{ label: 'Leaderboard', href: '/leaderboard', shortcut: 'l' },
{ label: 'Matches', href: '/matches' },
{ label: 'Usage', href: '/metrics', shortcut: 'm' }
] },
{ id: 'admin', label: 'Admin', admin: true, items: [
{ label: 'Users', href: '/settings/users' },
{ label: 'Keys', href: '/server-keys', shortcut: 'k' },
{ label: 'Settings', href: '/settings', shortcut: 'h', matches: path => path === '/settings' }
] }
{
id: 'fleet',
label: 'Fleet',
items: [
{
label: 'Servers',
href: '/',
shortcut: 'f',
matches: (path) => path === '/' || path.startsWith('/servers/')
},
{
label: 'Hosts',
href: '/hosts',
shortcut: 'o',
matches: (path) =>
path === '/hosts' || /^\/hosts\/(?!operations(?:\/|$))/.test(path)
},
{ label: 'Capacity', href: '/capacity' },
{ label: 'Autoscaling', href: '/autoscaling' },
{ label: 'Collections', href: '/collections' }
]
},
{
id: 'releases',
label: 'Releases',
items: [
{ label: 'Scripts', href: '/scripts', shortcut: 's' },
{ label: 'Configs', href: '/configs', shortcut: 'c' },
{ label: 'Plugins', href: '/plugins', shortcut: 'p' },
{ label: 'Game files', href: '/game-files' },
{ label: 'Betas', href: '/betas', shortcut: 'b' },
{ label: 'Campaigns', href: '/campaigns', shortcut: 'a' }
]
},
{
id: 'activity',
label: 'Activity',
items: [
{
label: 'Activity log',
href: '/activity',
shortcut: 'd',
matches: (path) =>
path === '/activity' || path === '/operations' || path === '/hosts/operations'
},
{
label: 'Provisioning',
href: '/provisioning',
matches: (path) => /^\/(provisioning|deployments)(\/|$)/.test(path)
},
{
label: 'Maintenance',
href: '/maintenance',
shortcut: 'w',
matches: (path) => /^\/(maintenance|watchdog)(\/|$)/.test(path)
}
]
},
{
id: 'players',
label: 'Players',
items: [
{ label: 'Leaderboard', href: '/leaderboard', shortcut: 'l' },
{ label: 'Matches', href: '/matches' },
{ label: 'Usage', href: '/metrics', shortcut: 'm' }
]
},
{
id: 'admin',
label: 'Admin',
admin: true,
items: [
{ label: 'Users', href: '/settings/users' },
{ label: 'Keys', href: '/server-keys', shortcut: 'k' },
{
label: 'Settings',
href: '/settings',
shortcut: 'h',
matches: (path) => path === '/settings'
}
]
}
];
export function visibleNavigation(role: UserRole): NavigationGroup[] {
return navigationGroups.filter(group => !group.admin || role === 'admin');
return navigationGroups.filter((group) => !group.admin || role === 'admin');
}
export function navigationItemActive(item: NavigationItem, path: string): boolean {
return item.matches ? item.matches(path) : path === item.href || path.startsWith(item.href + '/');
return item.matches
? item.matches(path)
: path === item.href || path.startsWith(item.href + '/');
}
export function navigationLocation(path: string, role: UserRole) {
for (const group of visibleNavigation(role)) {
const item = group.items.find(item => navigationItemActive(item, path));
const item = group.items.find((item) => navigationItemActive(item, path));
if (item) return { group, item };
}
return null;
src/lib/operation-flow.ts
+90 -29
View File
@@ -1,19 +1,42 @@
export interface ActivityEvent { id: number; created_at: string; message: string }
export interface OperationStep { label: string; state: 'pending' | 'current' | 'done' | 'failed' }
interface BetaServer {
stage: string; deployed_at: string | null; restarted_at: string | null;
healthy_at: string | null; match_seen_at: string | null;
export interface ActivityEvent {
id: number;
created_at: string;
message: string;
}
export interface OperationStep {
label: string;
state: 'pending' | 'current' | 'done' | 'failed';
}
export function betaFlow(beta: { state: string; wait_for_health: number; wait_for_match: number; testPlan?: boolean }, servers: BetaServer[]): OperationStep[] {
interface BetaServer {
stage: string;
deployed_at: string | null;
restarted_at: string | null;
healthy_at: string | null;
match_seen_at: string | null;
}
export function betaFlow(
beta: { state: string; wait_for_health: number; wait_for_match: number; testPlan?: boolean },
servers: BetaServer[]
): OperationStep[] {
const stages = ['deploying', 'awaiting-empty', 'restarting'];
const labels = ['Stage file changes', 'Wait for empty servers', 'Apply files and restart'];
if (beta.wait_for_health) { stages.push('awaiting-health'); labels.push('Verify health'); }
if (beta.wait_for_match) { stages.push('awaiting-match'); labels.push('Observe a completed match'); }
if (beta.testPlan) { stages.push('observing'); labels.push('Evaluate test requirements'); }
stages.push('ready'); labels.push('Review and promote');
const positions = servers.map(s => {
if (beta.wait_for_health) {
stages.push('awaiting-health');
labels.push('Verify health');
}
if (beta.wait_for_match) {
stages.push('awaiting-match');
labels.push('Observe a completed match');
}
if (beta.testPlan) {
stages.push('observing');
labels.push('Evaluate test requirements');
}
stages.push('ready');
labels.push('Review and promote');
const positions = servers.map((s) => {
if (s.stage !== 'failed') return Math.max(0, stages.indexOf(s.stage));
if (!s.deployed_at) return 0;
if (!s.restarted_at) return 2;
@@ -21,29 +44,67 @@ export function betaFlow(beta: { state: string; wait_for_health: number; wait_fo
if (beta.wait_for_match && !s.match_seen_at) return stages.indexOf('awaiting-match');
return stages.length - 1;
});
const current = beta.state === 'promoted' ? stages.length
: beta.state === 'ready-promote' ? stages.length - 1
: positions.length ? Math.min(...positions) : 0;
return labels.map((label, i) => ({ label, state: i < current ? 'done' : i === current
? beta.state === 'rolled-back' ? 'pending' : servers.some((s, j) => s.stage === 'failed' && positions[j] === i) ? 'failed' : 'current'
: 'pending' }));
const current =
beta.state === 'promoted'
? stages.length
: beta.state === 'ready-promote'
? stages.length - 1
: positions.length
? Math.min(...positions)
: 0;
return labels.map((label, i) => ({
label,
state:
i < current
? 'done'
: i === current
? beta.state === 'rolled-back'
? 'pending'
: servers.some((s, j) => s.stage === 'failed' && positions[j] === i)
? 'failed'
: 'current'
: 'pending'
}));
}
export function campaignFlow(servers: { server?: string; restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean,
files: { server: string; snapshot_pending: number; apply_state: string }[] = []): OperationStep[] {
const active = servers.filter(s => !['skipped', 'cancelled'].includes(s.restart_state));
const positions = active.map(s => {
export function campaignFlow(
servers: {
server?: string;
restart_state: string;
restarted_at: string | null;
files_changed: number;
}[],
finished: boolean,
files: { server: string; snapshot_pending: number; apply_state: string }[] = []
): OperationStep[] {
const active = servers.filter((s) => !['skipped', 'cancelled'].includes(s.restart_state));
const positions = active.map((s) => {
if (s.restart_state === 'done') return 5;
if (s.restarted_at) return 4;
const changes = files.filter(f => f.server === s.server && f.apply_state !== 'cancelled');
if (changes.some(f => f.apply_state === 'applying')) return changes.some(f => f.snapshot_pending) ? 1 : 2;
if (changes.length && changes.every(f => f.apply_state === 'applied')) return 3;
const changes = files.filter((f) => f.server === s.server && f.apply_state !== 'cancelled');
if (changes.some((f) => f.apply_state === 'applying'))
return changes.some((f) => f.snapshot_pending) ? 1 : 2;
if (changes.length && changes.every((f) => f.apply_state === 'applied')) return 3;
return 0;
});
const current = positions.length ? Math.min(...positions) : finished ? 5 : 0;
return ['Wait for empty servers and restart capacity', 'Capture rollback snapshots', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({
label, state: i < current ? 'done' : i !== current ? 'pending'
: active.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed'
: active.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current'
return [
'Wait for empty servers and restart capacity',
'Capture rollback snapshots',
'Apply file changes',
'Restart servers',
'Verify health'
].map((label, i) => ({
label,
state:
i < current
? 'done'
: i !== current
? 'pending'
: active.some((s, j) => s.restart_state === 'failed' && positions[j] === i)
? 'failed'
: active.some((s) => ['manual', 'held'].includes(s.restart_state))
? 'pending'
: 'current'
}));
}
src/lib/plutonium.ts
+34 -10
View File
@@ -52,9 +52,16 @@ export const PLUTONIUM_MANIFEST_URL = 'https://cdn.plutoniummod.com/updater/prod
export function parsePlutoniumManifest(value: unknown): number {
const manifest = value as { product?: unknown; revision?: unknown } | null;
if (!manifest || manifest.product !== 'plutonium-core-prod' || typeof manifest.revision !== 'number'
|| !Number.isSafeInteger(manifest.revision) || manifest.revision < 1) {
throw new Error('The production updater manifest did not contain a valid Plutonium revision.');
if (
!manifest ||
manifest.product !== 'plutonium-core-prod' ||
typeof manifest.revision !== 'number' ||
!Number.isSafeInteger(manifest.revision) ||
manifest.revision < 1
) {
throw new Error(
'The production updater manifest did not contain a valid Plutonium revision.'
);
}
return manifest.revision;
}
@@ -66,12 +73,20 @@ export function nextMaintenanceAt(time: string, now = new Date(), timeZone = 'UT
const [hour, minute] = time.split(':').map(Number);
const target = hour * 60 + minute;
const formatter = new Intl.DateTimeFormat('en-CA', {
timeZone, year: 'numeric', month: '2-digit', day: '2-digit',
hour: '2-digit', minute: '2-digit', hourCycle: 'h23'
timeZone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
hourCycle: 'h23'
});
function local(at: number) {
const parts = Object.fromEntries(formatter.formatToParts(at).map(p => [p.type, p.value]));
return { day: `${parts.year}-${parts.month}-${parts.day}`, minute: Number(parts.hour) * 60 + Number(parts.minute) };
const parts = Object.fromEntries(formatter.formatToParts(at).map((p) => [p.type, p.value]));
return {
day: `${parts.year}-${parts.month}-${parts.day}`,
minute: Number(parts.hour) * 60 + Number(parts.minute)
};
}
const today = local(now.getTime()).day;
const start = Math.floor(now.getTime() / 60_000) * 60_000;
@@ -83,9 +98,13 @@ export function nextMaintenanceAt(time: string, now = new Date(), timeZone = 'UT
const wall = local(at);
if (wall.day < today || wall.minute < target) continue;
const previous = days.get(wall.day);
if (!previous || wall.minute < previous.minute) days.set(wall.day, { at, minute: wall.minute });
if (!previous || wall.minute < previous.minute)
days.set(wall.day, { at, minute: wall.minute });
}
const next = [...days.values()].map(d => d.at).filter(at => at > now.getTime()).sort((a, b) => a - b)[0];
const next = [...days.values()]
.map((d) => d.at)
.filter((at) => at > now.getTime())
.sort((a, b) => a - b)[0];
if (next === undefined) throw new Error('Could not calculate the next maintenance time.');
return new Date(next).toISOString();
}
@@ -94,5 +113,10 @@ export function nextMaintenanceAt(time: string, now = new Date(), timeZone = 'UT
export function maintenanceTimeInZone(schedule: MaintenanceSchedule, timeZone: string): string {
if (schedule.timeZone === timeZone) return schedule.time;
const at = schedule.nextAt ?? nextMaintenanceAt(schedule.time, new Date(), schedule.timeZone);
return new Intl.DateTimeFormat('en-GB', { timeZone, hour: '2-digit', minute: '2-digit', hourCycle: 'h23' }).format(new Date(at));
return new Intl.DateTimeFormat('en-GB', {
timeZone,
hour: '2-digit',
minute: '2-digit',
hourCycle: 'h23'
}).format(new Date(at));
}
src/lib/provisioning-detail.ts
+40 -8
View File
@@ -1,15 +1,32 @@
import type { ProvisioningAction } from './provisioning.js';
import type { LaunchRecipeSummary } from './deployment-plan.js';
export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'prepare',
'container-action', 'cleanup', 'verify-container', 'verify-game',
'verify-map', 'verify-plutonium', 'verify-stopped'] as const;
export type ProvisioningPhase = typeof provisioningPhases[number];
export const provisioningPhases = [
'requested',
'eligibility',
'startup-queue',
'prepare',
'container-action',
'cleanup',
'verify-container',
'verify-game',
'verify-map',
'verify-plutonium',
'verify-stopped'
] as const;
export type ProvisioningPhase = (typeof provisioningPhases)[number];
export type EvidenceResult = 'pass' | 'waiting' | 'fail' | 'unknown';
export type EvidenceSource = 'agent' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null;
export interface ProvisioningCheck {
key: 'container' | 'game' | 'map' | 'plutonium-release' | 'plutonium-version' | 'occupancy' | 'stopped';
key:
| 'container'
| 'game'
| 'map'
| 'plutonium-release'
| 'plutonium-version'
| 'occupancy'
| 'stopped';
label: string;
result: EvidenceResult;
source: EvidenceSource;
@@ -46,7 +63,11 @@ export interface ProvisioningProgress {
blocker: ProvisioningBlocker | null;
checks: ProvisioningCheck[];
checkpoints: ProvisioningCheckpoint[];
command: { state: 'attempted' | 'accepted' | 'rejected' | 'uncertain'; at: string; message: string } | null;
command: {
state: 'attempted' | 'accepted' | 'rejected' | 'uncertain';
at: string;
message: string;
} | null;
recovery: RecoveryObservation | null;
recipeStatus: 'queued' | 'attempted' | null;
}
@@ -105,7 +126,14 @@ export interface ProvisioningDetail {
cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null;
backend?: 'native';
workerActivity?: WorkerActivity[];
agentOperation?: { id: string; hostId: string; state: string; updatedAt: string; createdAt: string; error: string | null } | null;
agentOperation?: {
id: string;
hostId: string;
state: string;
updatedAt: string;
createdAt: string;
error: string | null;
} | null;
deployment: ProvisioningDeployment;
events: ProvisioningEvent[];
activity: { total: number; latestId: number | null; failureExcerpt: ProvisioningEvent | null };
@@ -122,7 +150,11 @@ export interface ProvisioningDetail {
warningMessages: string[];
generatedAt: string;
}
export interface WorkerActivity { key: string; label: string; startedAt: string }
export interface WorkerActivity {
key: string;
label: string;
startedAt: string;
}
export interface ProvisioningEventPage {
events: ProvisioningEvent[];
total: number;
src/lib/provisioning-list.ts
+82 -20
View File
@@ -2,42 +2,104 @@ import type { ProvisioningAction } from './provisioning.js';
import type { ProvisioningDeployment } from './provisioning-detail.js';
import type { ProvisioningSource } from './provisioning-detail.js';
export const provisioningListStatuses = ['pending', 'running', 'awaiting-health', 'done', 'attention', 'failed', 'interrupted', 'warning', 'cancelled'] as const;
export const provisioningListSources = ['manual', 'campaign', 'beta', 'maintenance', 'recovery', 'autoscaling', 'rebalance', 'system'] as const;
export type ProvisioningListSource = typeof provisioningListSources[number];
export const provisioningListStatuses = [
'pending',
'running',
'awaiting-health',
'done',
'attention',
'failed',
'interrupted',
'warning',
'cancelled'
] as const;
export const provisioningListSources = [
'manual',
'campaign',
'beta',
'maintenance',
'recovery',
'autoscaling',
'rebalance',
'system'
] as const;
export type ProvisioningListSource = (typeof provisioningListSources)[number];
export const provisioningSourceLabels: Record<ProvisioningListSource, string> = {
manual: 'Manual', campaign: 'Campaign', beta: 'Beta', maintenance: 'Maintenance',
recovery: 'Recovery', autoscaling: 'Autoscaling', rebalance: 'Rebalance', system: 'System'
manual: 'Manual',
campaign: 'Campaign',
beta: 'Beta',
maintenance: 'Maintenance',
recovery: 'Recovery',
autoscaling: 'Autoscaling',
rebalance: 'Rebalance',
system: 'System'
};
export const provisioningSourceKinds = {
campaign: ['campaign'], beta: ['beta'], maintenance: ['maintenance'],
recovery: ['watchdog'], autoscaling: ['autoscaling'], rebalance: ['rebalance']
} as const satisfies Record<Exclude<ProvisioningListSource, 'manual' | 'system'>, readonly ProvisioningSource['kind'][]>;
campaign: ['campaign'],
beta: ['beta'],
maintenance: ['maintenance'],
recovery: ['watchdog'],
autoscaling: ['autoscaling'],
rebalance: ['rebalance']
} as const satisfies Record<
Exclude<ProvisioningListSource, 'manual' | 'system'>,
readonly ProvisioningSource['kind'][]
>;
/** Record titles and incident summaries belong in expanded details, not the source column. */
export function provisioningSourceLabel(row: Pick<ProvisioningListRow, 'sources' | 'triggered_by'>): string {
export function provisioningSourceLabel(
row: Pick<ProvisioningListRow, 'sources' | 'triggered_by'>
): string {
const kind = row.sources?.[0]?.kind;
return provisioningSourceLabels[kind === 'watchdog' ? 'recovery' : kind ?? (row.triggered_by === 'ui' ? 'manual' : 'system')];
return provisioningSourceLabels[
kind === 'watchdog'
? 'recovery'
: (kind ?? (row.triggered_by === 'ui' ? 'manual' : 'system'))
];
}
export function provisioningMatchesSource(row: Pick<ProvisioningListRow, 'sources' | 'triggered_by'>, source: ProvisioningListSource): boolean {
export function provisioningMatchesSource(
row: Pick<ProvisioningListRow, 'sources' | 'triggered_by'>,
source: ProvisioningListSource
): boolean {
const sources = row.sources ?? [];
if (source === 'manual' || source === 'system') return !sources.length && (source === 'manual' ? row.triggered_by === 'ui' : row.triggered_by !== 'ui');
if (source === 'manual' || source === 'system')
return (
!sources.length &&
(source === 'manual' ? row.triggered_by === 'ui' : row.triggered_by !== 'ui')
);
const kinds: readonly ProvisioningSource['kind'][] = provisioningSourceKinds[source];
return sources.some(item => kinds.includes(item.kind));
return sources.some((item) => kinds.includes(item.kind));
}
export type ProvisioningListRow = Pick<ProvisioningDeployment,
'id' | 'server' | 'server_deleted_at' | 'action' | 'state' | 'stage' | 'started_at' |
'finished_at' | 'triggered_by' | 'requested_by' | 'error' | 'warnings' | 'server_exists'> & {
canCancelRemoval: number; canCancel?: number; sources?: ProvisioningSource[];
phase?: string | null; phaseStartedAt?: string | null; waitCode?: string | null;
};
export type ProvisioningListRow = Pick<
ProvisioningDeployment,
| 'id'
| 'server'
| 'server_deleted_at'
| 'action'
| 'state'
| 'stage'
| 'started_at'
| 'finished_at'
| 'triggered_by'
| 'requested_by'
| 'error'
| 'warnings'
| 'server_exists'
> & {
canCancelRemoval: number;
canCancel?: number;
sources?: ProvisioningSource[];
phase?: string | null;
phaseStartedAt?: string | null;
waitCode?: string | null;
};
export interface ProvisioningListOptions {
view?: 'active' | 'history' | 'all';
action?: ProvisioningAction;
status?: typeof provisioningListStatuses[number];
status?: (typeof provisioningListStatuses)[number];
source?: ProvisioningListSource;
server?: string;
limit?: number;
src/lib/provisioning.ts
+23 -6
View File
@@ -1,17 +1,34 @@
export const provisioningActions = ['create', 'start', 'restart', 'rebuild', 'stop', 'destroy'] as const;
export type ProvisioningAction = typeof provisioningActions[number];
export const provisioningActions = [
'create',
'start',
'restart',
'rebuild',
'stop',
'destroy'
] as const;
export type ProvisioningAction = (typeof provisioningActions)[number];
export type RuntimeAction = Exclude<ProvisioningAction, 'create' | 'destroy'>;
export const actionLabels: Record<ProvisioningAction, string> = {
create: 'Create', start: 'Start', restart: 'Restart', rebuild: 'Rebuild', stop: 'Stop', destroy: 'Destroy'
create: 'Create',
start: 'Start',
restart: 'Restart',
rebuild: 'Rebuild',
stop: 'Stop',
destroy: 'Destroy'
};
export const actionCompleted: Record<ProvisioningAction, string> = {
create: 'Server creation completed', start: 'Server started', restart: 'Server restarted',
rebuild: 'Server rebuilt', stop: 'Server stopped', destroy: 'Server destroyed'
create: 'Server creation completed',
start: 'Server started',
restart: 'Server restarted',
rebuild: 'Server rebuilt',
stop: 'Server stopped',
destroy: 'Server destroyed'
};
export const runtimeActions: RuntimeAction[] = ['start', 'restart', 'rebuild', 'stop'];
export const runtimeDescriptions: Record<RuntimeAction, string> = {
start: 'Start the existing container and verify its health.',
restart: 'Restart the container after the server is empty, then verify its health.',
rebuild: 'Remove the instance and delete all of its server files, then create a fresh instance in the same server slot and port with the latest promoted setup. No instance data is retained. Uses the key saved separately in Keys. Waits until empty, then verifies health and map.',
rebuild:
'Remove the instance and delete all of its server files, then create a fresh instance in the same server slot and port with the latest promoted setup. No instance data is retained. Uses the key saved separately in Keys. Waits until empty, then verifies health and map.',
stop: 'Stop the container after the server is empty. Its files, key, and registration are retained so you can start it again.'
};
src/lib/rebalancing.ts
+57 -17
View File
@@ -4,15 +4,26 @@ export interface RebalanceSettings {
maxMoves: number;
}
export interface RebalanceHostOption { id: string; name: string; eligible: boolean; reason: string | null }
export interface RebalanceHostOption {
id: string;
name: string;
eligible: boolean;
reason: string | null;
}
export interface RebalanceHostProjection {
id: string; name: string;
id: string;
name: string;
before: { cpuPercent: number; memoryPercent: number; instances: number };
after: { cpuPercent: number; memoryPercent: number; instances: number };
}
export interface RebalanceCandidate {
server: string; instanceId: string; generation: number;
sourceHostId: string; sourceHostName: string; targetHostId: string; targetHostName: string;
server: string;
instanceId: string;
generation: number;
sourceHostId: string;
sourceHostName: string;
targetHostId: string;
targetHostName: string;
/** Estimated reduction in the source/destination pair's peak utilization, in percentage points. */
improvement: number;
}
@@ -23,26 +34,55 @@ export interface RebalancePlan {
warnings: string[];
}
export interface RebalancePreview extends RebalancePlan {
id: string; settings: RebalanceSettings; createdAt: string; expiresAt: string;
id: string;
settings: RebalanceSettings;
createdAt: string;
expiresAt: string;
}
export interface RebalanceMove extends RebalanceCandidate {
id: string; state: 'planned' | 'moving' | 'done' | 'skipped' | 'failed';
operationId: string | null; message: string; updatedAt: string;
id: string;
state: 'planned' | 'moving' | 'done' | 'skipped' | 'failed';
operationId: string | null;
message: string;
updatedAt: string;
}
export interface RebalanceRun {
id: string; settings: RebalanceSettings;
id: string;
settings: RebalanceSettings;
state: 'running' | 'stopping' | 'done' | 'stopped' | 'failed';
createdAt: string; updatedAt: string; message: string; moves: RebalanceMove[];
createdAt: string;
updatedAt: string;
message: string;
moves: RebalanceMove[];
}
export interface RebalanceReport {
hosts: RebalanceHostOption[];
runs: RebalanceRun[];
}
export interface RebalanceReport { hosts: RebalanceHostOption[]; runs: RebalanceRun[] }
export function rebalanceSettings(input: unknown): RebalanceSettings {
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide rebalance settings.');
if (!input || typeof input !== 'object' || Array.isArray(input))
throw new Error('Provide rebalance settings.');
const value = input as Record<string, unknown>;
if (!Array.isArray(value.hostIds) || value.hostIds.length < 2 || value.hostIds.length > 100
|| value.hostIds.some(id => typeof id !== 'string' || !id || id.length > 100)
|| new Set(value.hostIds).size !== value.hostIds.length) throw new Error('Choose between 2 and 100 different hosts.');
if (!['all', 't5', 't6'].includes(value.game as string)) throw new Error('Choose a supported game.');
if (!Number.isInteger(value.maxMoves) || (value.maxMoves as number) < 1 || (value.maxMoves as number) > 20) throw new Error('Choose a move limit between 1 and 20.');
return { hostIds: [...value.hostIds as string[]].sort(), game: value.game as RebalanceSettings['game'], maxMoves: value.maxMoves as number };
if (
!Array.isArray(value.hostIds) ||
value.hostIds.length < 2 ||
value.hostIds.length > 100 ||
value.hostIds.some((id) => typeof id !== 'string' || !id || id.length > 100) ||
new Set(value.hostIds).size !== value.hostIds.length
)
throw new Error('Choose between 2 and 100 different hosts.');
if (!['all', 't5', 't6'].includes(value.game as string))
throw new Error('Choose a supported game.');
if (
!Number.isInteger(value.maxMoves) ||
(value.maxMoves as number) < 1 ||
(value.maxMoves as number) > 20
)
throw new Error('Choose a move limit between 1 and 20.');
return {
hostIds: [...(value.hostIds as string[])].sort(),
game: value.game as RebalanceSettings['game'],
maxMoves: value.maxMoves as number
};
}
src/lib/script-library.ts
+53 -19
View File
@@ -2,20 +2,34 @@ import { T5_ZOMBIE_MAPS, T6_ZOMBIE_MAPS } from './map-aliases.js';
import { gameTitle } from './ui/server-labels.js';
export interface LibraryVersion {
id: number; version: number; sha256: string; size: number;
state: 'untested' | 'tested' | 'test-failed'; uploaded_at: string;
id: number;
version: number;
sha256: string;
size: number;
state: 'untested' | 'tested' | 'test-failed';
uploaded_at: string;
}
export interface LibraryScript {
id: number; name: string; target_rel_path: string;
scope_type: 'game' | 'map' | null; scope_value: string | null;
id: number;
name: string;
target_rel_path: string;
scope_type: 'game' | 'map' | null;
scope_value: string | null;
current_version_id: number | null;
currentVersion: LibraryVersion | null; latestVersion: LibraryVersion | null;
versionCount: number; servers: string[]; runningBetaId: number | null;
currentVersion: LibraryVersion | null;
latestVersion: LibraryVersion | null;
versionCount: number;
servers: string[];
runningBetaId: number | null;
}
export type ScriptStatus = 'untested' | 'testing' | 'tested' | 'failed' | 'promoted' | 'empty';
export const statusLabels: Record<ScriptStatus, string> = {
untested: 'Needs testing', testing: 'Beta active', tested: 'Test passed',
failed: 'Test failed', promoted: 'Promoted', empty: 'No versions'
untested: 'Needs testing',
testing: 'Beta active',
tested: 'Test passed',
failed: 'Test failed',
promoted: 'Promoted',
empty: 'No versions'
};
export function scriptStatus(s: LibraryScript): ScriptStatus {
if (s.runningBetaId !== null) return 'testing';
@@ -24,17 +38,29 @@ export function scriptStatus(s: LibraryScript): ScriptStatus {
return s.latestVersion.state === 'test-failed' ? 'failed' : s.latestVersion.state;
}
export function canTest(s: LibraryScript): boolean {
return s.runningBetaId === null && s.latestVersion !== null && s.latestVersion.id !== s.current_version_id;
return (
s.runningBetaId === null &&
s.latestVersion !== null &&
s.latestVersion.id !== s.current_version_id
);
}
export function mapLabel(canonical: string): string {
const matches = [...T5_ZOMBIE_MAPS, ...T6_ZOMBIE_MAPS].filter(m => m.canonical === canonical);
return matches.length > 1 ? `${matches[0].friendly} + ${matches.length - 1} variants` : matches[0]?.friendly ?? canonical;
const matches = [...T5_ZOMBIE_MAPS, ...T6_ZOMBIE_MAPS].filter((m) => m.canonical === canonical);
return matches.length > 1
? `${matches[0].friendly} + ${matches.length - 1} variants`
: (matches[0]?.friendly ?? canonical);
}
export function scopeLabel(s: Pick<LibraryScript, 'scope_type' | 'scope_value'>): string {
return s.scope_type === 'game' ? gameTitle(s.scope_value)
: s.scope_type === 'map' ? mapLabel(s.scope_value ?? '') : 'Unscoped';
return s.scope_type === 'game'
? gameTitle(s.scope_value)
: s.scope_type === 'map'
? mapLabel(s.scope_value ?? '')
: 'Unscoped';
}
export interface BetaSelection {
scriptId: number;
versionId: number;
}
export interface BetaSelection { scriptId: number; versionId: number; }
export function betaSelectionHref(selections: BetaSelection[]): string {
const query = new URLSearchParams();
for (const s of selections) query.append('artifact', `${s.scriptId}:${s.versionId}`);
@@ -42,13 +68,21 @@ export function betaSelectionHref(selections: BetaSelection[]): string {
}
export function parseBetaSelection(query: URLSearchParams): BetaSelection[] {
const values = query.getAll('artifact');
if (!values.length && query.has('pluginId')) values.push(`${query.get('pluginId')}:${query.get('versionId')}`);
if (!values.length && query.has('pluginId'))
values.push(`${query.get('pluginId')}:${query.get('versionId')}`);
const seen = new Set<number>();
return values.map(value => {
if (!/^[1-9]\d*:[1-9]\d*$/.test(value)) throw new Error('The beta selection link is invalid. Choose artifacts below.');
return values.map((value) => {
if (!/^[1-9]\d*:[1-9]\d*$/.test(value))
throw new Error('The beta selection link is invalid. Choose artifacts below.');
const [scriptId, versionId] = value.split(':').map(Number);
if (!Number.isSafeInteger(scriptId) || !Number.isSafeInteger(versionId) || seen.has(scriptId)) {
throw new Error('The beta selection link contains invalid or repeated artifacts. Choose artifacts below.');
if (
!Number.isSafeInteger(scriptId) ||
!Number.isSafeInteger(versionId) ||
seen.has(scriptId)
) {
throw new Error(
'The beta selection link contains invalid or repeated artifacts. Choose artifacts below.'
);
}
seen.add(scriptId);
return { scriptId, versionId };
src/lib/server-capacity.ts
+13 -4
View File
@@ -6,11 +6,20 @@ export interface PlayerCapacityObservation {
}
/** A newer roster must never renew the capacity reported by an older process. */
export function currentCapacity(server: Partial<PlayerCapacityObservation> & { telemetry_after?: string | null }, now: number): number | null {
export function currentCapacity(
server: Partial<PlayerCapacityObservation> & { telemetry_after?: string | null },
now: number
): number | null {
const value = server.capacity;
const observedAt = server.capacity_observed_at;
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0
|| !freshStatusTimestamp(observedAt ?? null, now)) return null;
if (server.telemetry_after && !(Date.parse(observedAt!) > Date.parse(server.telemetry_after))) return null;
if (
typeof value !== 'number' ||
!Number.isSafeInteger(value) ||
value <= 0 ||
!freshStatusTimestamp(observedAt ?? null, now)
)
return null;
if (server.telemetry_after && !(Date.parse(observedAt!) > Date.parse(server.telemetry_after)))
return null;
return value;
}
src/lib/server-observation.ts
+114 -27
View File
@@ -36,51 +36,112 @@ export interface ServerObservation extends GameObservation {
}
/** Keep a first transient monitoring failure in diagnostics without demoting health. */
export function rconHealthError(status: Pick<GameObservation, 'last_error' | 'rcon_failures'>): string | null {
export function rconHealthError(
status: Pick<GameObservation, 'last_error' | 'rcon_failures'>
): string | null {
const reason = status.last_error?.split(':')[0];
return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error') ? null : status.last_error;
return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error')
? null
: status.last_error;
}
export function freshTelemetry(report: GameTelemetry | undefined, now: number, after?: string | null): report is GameTelemetry {
export function freshTelemetry(
report: GameTelemetry | undefined,
now: number,
after?: string | null
): report is GameTelemetry {
if (!report || !freshStatusTimestamp(report.receivedAt, now)) return false;
if (after && (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))) return false;
try { parseGameEvent(report); return true; } catch { return false; }
if (
after &&
(!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))
)
return false;
try {
parseGameEvent(report);
return true;
} catch {
return false;
}
}
function playerMatchGuid(guid: string, game: string | null | undefined, source: 'rcon' | 'telemetry'): string {
function playerMatchGuid(
guid: string,
game: string | null | undefined,
source: 'rcon' | 'telemetry'
): string {
if (game !== 't6') return guid;
// T6 status prints hexadecimal account IDs; telemetry sends decimal IDs.
// Keep integer precision for IDs beyond Number.MAX_SAFE_INTEGER.
if (source === 'rcon' && /^(?:0x)?[0-9a-f]+$/i.test(guid)) return BigInt(`0x${guid.replace(/^0x/i, '')}`).toString();
if (source === 'rcon' && /^(?:0x)?[0-9a-f]+$/i.test(guid))
return BigInt(`0x${guid.replace(/^0x/i, '')}`).toString();
if (source === 'telemetry' && /^\d+$/.test(guid)) return BigInt(guid).toString();
return guid;
}
/** Display policy: use one current roster/count source, keeping raw RCON for diagnostics/fallback. */
export function selectServerObservation<T extends ServerObservation>(server: T, report: GameTelemetry | undefined, now: number): T {
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
export function selectServerObservation<T extends ServerObservation>(
server: T,
report: GameTelemetry | undefined,
now: number
): T {
if (
server.telemetry &&
(!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))
)
report = server.telemetry;
const raw = server.rcon_status ?? server;
const result: T = { ...server, ...raw, rcon_status: raw, last_error: rconHealthError(raw), telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
const result: T = {
...server,
...raw,
rcon_status: raw,
last_error: rconHealthError(raw),
telemetry: report,
status_source: raw.last_seen_at ? 'rcon' : null
};
const containerUnknown = server.container_state === 'unknown';
if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) {
return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null };
if (
['provisioning', 'starting'].includes(server.health) ||
(server.container_state && server.container_state !== 'running' && !containerUnknown) ||
(!server.container_state && server.health === 'offline')
) {
return {
...result,
health: server.health,
last_error: null,
last_players: null,
roster: null,
status_source: null
};
}
const checked = server.container_checked_at ?? server.last_checked_at ?? raw.last_seen_at;
const containerFresh = server.container_state === 'running' && freshStatusTimestamp(checked ?? null, now);
const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
const containerFresh =
server.container_state === 'running' && freshStatusTimestamp(checked ?? null, now);
const after =
!server.telemetry_after ||
Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !result.last_error;
if (containerFresh && report?.server === server.name && freshTelemetry(report, now, server.telemetry_after)) {
if (
containerFresh &&
report?.server === server.name &&
freshTelemetry(report, now, server.telemetry_after)
) {
const players = report.players.length;
// Telemetry owns roster membership. Supplement missing pings only with
// unambiguous RCON identities from the current map and server lifecycle.
const pings = new Map<string, number | null>();
const guidKey = (guid: string, source: 'rcon' | 'telemetry') => playerMatchGuid(guid, server.game, source);
const guidKey = (guid: string, source: 'rcon' | 'telemetry') =>
playerMatchGuid(guid, server.game, source);
if (fresh && raw.last_map === report.map) {
for (const player of raw.roster ?? []) {
const ping = player.ping;
if (player.guid) {
const guid = guidKey(player.guid, 'rcon');
pings.set(guid, pings.has(guid) || ping === null || !Number.isSafeInteger(ping) || ping < 0 ? null : ping);
pings.set(
guid,
pings.has(guid) || ping === null || !Number.isSafeInteger(ping) || ping < 0
? null
: ping
);
}
}
}
@@ -89,23 +150,49 @@ export function selectServerObservation<T extends ServerObservation>(server: T,
const timing = report.timing?.server ?? {
activeSince: server.active_since ?? report.receivedAt,
inactiveSince: server.inactive_since ?? report.receivedAt,
gameStartedAt: report.event === 'game_start' ? report.receivedAt : server.game_started_at,
gameEndedAt: report.event === 'game_start' ? null : report.event === 'game_end' ? report.receivedAt : server.game_ended_at
gameStartedAt:
report.event === 'game_start' ? report.receivedAt : server.game_started_at,
gameEndedAt:
report.event === 'game_start'
? null
: report.event === 'game_end'
? report.receivedAt
: server.game_ended_at
};
return { ...result, health: 'online', status_source: 'telemetry', last_players: players,
last_map: report.map, last_seen_at: report.receivedAt, last_error: null,
roster: report.players.map(player => ({ num: null, score: 'score' in player ? player.score : null,
ping: player.ping ?? pings.get(guidKey(player.guid, 'telemetry')) ?? null, guid: player.guid, name: player.name })),
return {
...result,
health: 'online',
status_source: 'telemetry',
last_players: players,
last_map: report.map,
last_seen_at: report.receivedAt,
last_error: null,
roster: report.players.map((player) => ({
num: null,
score: 'score' in player ? player.score : null,
ping: player.ping ?? pings.get(guidKey(player.guid, 'telemetry')) ?? null,
guid: player.guid,
name: player.name
})),
active_since: players > 0 ? timing.activeSince : null,
inactive_since: players === 0 ? timing.inactiveSince : null,
game_started_at: players === 0 ? null : timing.gameStartedAt,
game_ended_at: players === 0 ? null : timing.gameEndedAt };
game_ended_at: players === 0 ? null : timing.gameEndedAt
};
}
// A current authenticated RCON reply can establish game liveness when
// inspection is unavailable, without claiming the container was inspected.
const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh;
const age = now - Date.parse(raw.last_seen_at ?? '');
const health = fresh && !containerExpired ? 'online'
: after && age >= 0 && age <= 120_000 ? 'stale' : 'offline';
return { ...result, health, last_error: health === 'online' ? result.last_error : raw.last_error };
const health =
fresh && !containerExpired
? 'online'
: after && age >= 0 && age <= 120_000
? 'stale'
: 'offline';
return {
...result,
health,
last_error: health === 'online' ? result.last_error : raw.last_error
};
}
src/lib/server/activity.ts
+88 -24
View File
@@ -1,6 +1,12 @@
import { hostOperationTitles } from '../host-operation-labels.js';
import type Database from 'better-sqlite3';
import { normalizeActivityQuery, type ActivityPage, type ActivityQuery, type ActivityRow, type ActivitySource } from '../activity.js';
import {
normalizeActivityQuery,
type ActivityPage,
type ActivityQuery,
type ActivityRow,
type ActivitySource
} from '../activity.js';
import { serverTitle } from '../ui/server-labels.js';
import { db } from './db/index.js';
import { ensureHostSchema } from './hosts/schema.js';
@@ -9,16 +15,22 @@ import { importantOperationSql } from './hosts/observations.js';
const initialized = new WeakSet<Database.Database>();
const quote = (value: string) => `'${value.replaceAll("'", "''")}'`;
const labelCase = (column: string, labels: Record<string, string>, fallback: string) =>
`CASE ${column} ${Object.entries(labels).map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`).join(' ')} ELSE ${quote(fallback)} END`;
`CASE ${column} ${Object.entries(labels)
.map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`)
.join(' ')} ELSE ${quote(fallback)} END`;
// Old attempts may no longer be the native_provisioning pointer. Their stable
// idempotency keys still identify the same retained controller operation.
const lifecycleKey = `CASE
WHEN substr(o.idempotency_key,-6)=':stage' THEN substr(o.idempotency_key,1,length(o.idempotency_key)-6)
${[':migration-create:', ':migrate-remove:'].map(marker => `WHEN instr(o.idempotency_key,${quote(marker)})>0
${[':migration-create:', ':migrate-remove:']
.map(
(marker) => `WHEN instr(o.idempotency_key,${quote(marker)})>0
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length})<>''
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length}) NOT GLOB '*[^0-9]*'
THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`).join('\n')}
THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`
)
.join('\n')}
ELSE o.idempotency_key END`;
/** One row per user-level operation or finding; no command payloads or observations are read. */
@@ -128,19 +140,50 @@ normalized AS (
)
`;
type SqlRow = Omit<ActivityRow, 'source' | 'sources' | 'href'> & { sources: string; route: string; recordId: string; secondaryHostId: string | null };
const sourceOrder: ActivitySource[] = ['maintenance', 'recovery', 'campaign', 'beta', 'autoscaling', 'provisioning', 'host'];
type SqlRow = Omit<ActivityRow, 'source' | 'sources' | 'href'> & {
sources: string;
route: string;
recordId: string;
secondaryHostId: string | null;
};
const sourceOrder: ActivitySource[] = [
'maintenance',
'recovery',
'campaign',
'beta',
'autoscaling',
'provisioning',
'host'
];
// Use the same membership rule for Maintenance history and its exclusion from the main log.
const maintenanceMembershipSql = "(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)";
const maintenanceMembershipSql =
"(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)";
function present(row: SqlRow): ActivityRow {
const sources = sourceOrder.filter(source => row.sources.includes(`|${source}|`));
const href = row.route === 'maintenance-job' ? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}`
: row.route === 'recovery' ? `/maintenance/${encodeURIComponent(row.recordId)}`
: row.route === 'host' ? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all`
: `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`;
return { id: row.id, at: row.at, type: row.type, source: sources[0] ?? 'provisioning', sources,
state: row.state, rawState: row.rawState, title: row.title, detail: row.detail,
server: row.server, serverDeletedAt: row.serverDeletedAt, hostId: row.hostId, hostName: row.hostName, href };
const sources = sourceOrder.filter((source) => row.sources.includes(`|${source}|`));
const href =
row.route === 'maintenance-job'
? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}`
: row.route === 'recovery'
? `/maintenance/${encodeURIComponent(row.recordId)}`
: row.route === 'host'
? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all`
: `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`;
return {
id: row.id,
at: row.at,
type: row.type,
source: sources[0] ?? 'provisioning',
sources,
state: row.state,
rawState: row.rawState,
title: row.title,
detail: row.detail,
server: row.server,
serverDeletedAt: row.serverDeletedAt,
hostId: row.hostId,
hostName: row.hostName,
href
};
}
/** All predicates, totals and pagination operate on the same retained SQL union. */
@@ -149,7 +192,9 @@ export function activityHistory(input: ActivityQuery = {}, now = Date.now()): Ac
const handle = db();
ensureHostSchema(handle);
if (!initialized.has(handle)) {
handle.function('activity_server_title', { deterministic: true }, (name: unknown) => typeof name === 'string' ? serverTitle(name) : '');
handle.function('activity_server_title', { deterministic: true }, (name: unknown) =>
typeof name === 'string' ? serverTitle(name) : ''
);
initialized.add(handle);
}
const conditions: string[] = [];
@@ -158,25 +203,44 @@ export function activityHistory(input: ActivityQuery = {}, now = Date.now()): Ac
if (query.excludeProvisioning) conditions.push("route <> 'provisioning' AND type <> 'restart'");
if (query.excludeMaintenance) conditions.push(`NOT ${maintenanceMembershipSql}`);
const bindings: Record<string, string | number> = {};
for (const key of ['server', 'type', 'state'] as const) if (query[key]) { conditions.push(`${key}=@${key}`); bindings[key] = query[key]; }
if (query.host) { conditions.push('(hostId=@host OR secondaryHostId=@host)'); bindings.host = query.host; }
for (const key of ['server', 'type', 'state'] as const)
if (query[key]) {
conditions.push(`${key}=@${key}`);
bindings[key] = query[key];
}
if (query.host) {
conditions.push('(hostId=@host OR secondaryHostId=@host)');
bindings.host = query.host;
}
if (query.source) {
// Maintenance is the user-facing umbrella; Recovery remains available as a narrower filter.
conditions.push(query.source === 'maintenance' ? maintenanceMembershipSql : 'instr(sources,@source)>0');
conditions.push(
query.source === 'maintenance' ? maintenanceMembershipSql : 'instr(sources,@source)>0'
);
if (query.source !== 'maintenance') bindings.source = `|${query.source}|`;
}
if (query.days) { conditions.push('julianday(at)>=julianday(@since)'); bindings.since = new Date(now - query.days * 86_400_000).toISOString(); }
if (query.days) {
conditions.push('julianday(at)>=julianday(@since)');
bindings.since = new Date(now - query.days * 86_400_000).toISOString();
}
if (query.q) {
conditions.push("instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0");
conditions.push(
"instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0"
);
bindings.q = query.q.toLocaleLowerCase();
}
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
const limit = query.limit!;
return handle.transaction(() => {
const { total } = handle.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`).get(bindings) as { total: number };
const { total } = handle
.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`)
.get(bindings) as { total: number };
const offset = Math.min(query.offset!, Math.max(0, Math.ceil(total / limit) - 1) * limit);
const rows = handle.prepare(`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId
FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`)
const rows = handle
.prepare(
`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId
FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`
)
.all({ ...bindings, limit, offset }) as SqlRow[];
return { rows: rows.map(present), total, limit, offset, query: { ...query, offset } };
})();
src/lib/server/auth/access.ts
+50 -16
View File
@@ -1,8 +1,10 @@
import type { UserRole } from '../../auth.js';
export type RoutePermission = 'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin';
export type RoutePermission =
'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin';
const within = (route: string, base: string): boolean => route === base || route.startsWith(base + '/');
const within = (route: string, base: string): boolean =>
route === base || route.startsWith(base + '/');
const readMethod = (method: string): boolean => method === 'GET' || method === 'HEAD';
/** Classify SvelteKit's resolved route ID, never a user-supplied URL pathname.
@@ -15,32 +17,63 @@ export function routePermission(routeId: string | null, method: string): RoutePe
if (routeId === null) return reading ? 'viewer' : 'operator';
// These cannot occur in this application's resolved route IDs. Fail closed
// if a future caller accidentally supplies a raw or noncanonical pathname.
if (!routeId.startsWith('/') || /[%\\?#\x00-\x1f\x7f]/.test(routeId)
|| routeId !== '/' && routeId.split('/').slice(1).some(part => !part || part === '.' || part === '..')) return 'admin';
if (
!routeId.startsWith('/') ||
// eslint-disable-next-line no-control-regex -- Reject control characters in resolved route IDs.
/[%\\?#\x00-\x1f\x7f]/.test(routeId) ||
(routeId !== '/' &&
routeId
.split('/')
.slice(1)
.some((part) => !part || part === '.' || part === '..'))
)
return 'admin';
if (within(routeId, '/api/agent')) return 'machine';
if (routeId === '/api/telemetry' && method === 'POST') return 'machine';
if (routeId === '/api/leaderboard' && reading) return 'public';
if ((routeId === '/login' || routeId === '/setup') && reading) return 'public';
if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST') return 'public';
if (routeId === '/account' || ['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId)) return 'authenticated';
if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST')
return 'public';
if (
routeId === '/account' ||
['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId)
)
return 'authenticated';
if (within(routeId, '/api/users') || within(routeId, '/settings') || within(routeId, '/api/settings') || within(routeId, '/api/auth')
|| within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin';
if (
within(routeId, '/api/users') ||
within(routeId, '/settings') ||
within(routeId, '/api/settings') ||
within(routeId, '/api/auth') ||
within(routeId, '/api/server-keys') ||
within(routeId, '/server-keys') ||
within(routeId, '/keys')
)
return 'admin';
if (!reading && ['/api/hosts', '/api/game-files'].some(base => within(routeId, base))) return 'admin';
if (!reading && ['/api/hosts', '/api/game-files'].some((base) => within(routeId, base)))
return 'admin';
if (routeId === '/api/provisioning/[id]/override') return 'admin';
// These endpoints mix operational actions with global policy and schedule
// changes. Per-incident/job actions remain available to operators.
if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) return 'admin';
if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId))
return 'admin';
// Summaries remain visible to viewers; raw configuration, historical file
// bytes, and unfiltered diagnostics may contain runtime credentials.
if (routeId === '/configs/[id]' || routeId === '/api/configs/[id]' || routeId === '/api/configs/inspect'
|| routeId === '/api/campaigns/[id]/file/[fileId]' || routeId === '/api/deployments/[id]/file/[fileId]'
|| routeId === '/api/betas/[id]/revisions/[revision]/files/[scriptId]'
|| routeId === '/api/servers/[name]/logs/[file]' || routeId === '/api/servers/[name]/telemetry-requests'
|| routeId === '/api/servers/[name]/rcon-diagnostics') return 'operator';
if (
routeId === '/configs/[id]' ||
routeId === '/api/configs/[id]' ||
routeId === '/api/configs/inspect' ||
routeId === '/api/campaigns/[id]/file/[fileId]' ||
routeId === '/api/deployments/[id]/file/[fileId]' ||
routeId === '/api/betas/[id]/revisions/[revision]/files/[scriptId]' ||
routeId === '/api/servers/[name]/logs/[file]' ||
routeId === '/api/servers/[name]/telemetry-requests' ||
routeId === '/api/servers/[name]/rcon-diagnostics'
)
return 'operator';
return reading ? 'viewer' : 'operator';
}
@@ -49,7 +82,8 @@ export function routePermission(routeId: string | null, method: string): RoutePe
export function roleAllows(role: UserRole, permission: RoutePermission): boolean {
if (!['viewer', 'operator', 'admin'].includes(role)) return false;
if (permission === 'machine') return false;
if (permission === 'public' || permission === 'authenticated' || permission === 'viewer') return true;
if (permission === 'public' || permission === 'authenticated' || permission === 'viewer')
return true;
if (permission === 'operator') return role === 'operator' || role === 'admin';
return permission === 'admin' && role === 'admin';
}
src/lib/server/auth/guard.ts
+174 -52
View File
@@ -1,112 +1,234 @@
import { error, json, redirect, type Handle } from '@sveltejs/kit';
import { readSession, readSessionHash, recordAudit, setupRequired } from './index.js';
import { authBody, authError, requireSameOrigin, requireUserTransport, sessionCookie } from './http.js';
import {
authBody,
authError,
requireSameOrigin,
requireUserTransport,
sessionCookie
} from './http.js';
import { roleAllows, routePermission, type RoutePermission } from './access.js';
import { withRequestUsername } from './request-context.js';
function publicAsset(routeId: string | null, path: string, method: string) {
return routeId === null && ['GET','HEAD'].includes(method) &&
(path.startsWith('/_app/') || path === '/favicon.svg' || ['/agent/ec-agent.py','/agent/ec-agent-update.py','/agent/install.sh','/agent/manifest.json'].includes(path));
return (
routeId === null &&
['GET', 'HEAD'].includes(method) &&
(path.startsWith('/_app/') ||
path === '/favicon.svg' ||
[
'/agent/ec-agent.py',
'/agent/ec-agent-update.py',
'/agent/install.sh',
'/agent/manifest.json'
].includes(path))
);
}
/** Streaming responses must lose access when the underlying session does. */
export function protectEventStream(response: Response, tokenHash: string, permission: RoutePermission, signal: AbortSignal): Response {
export function protectEventStream(
response: Response,
tokenHash: string,
permission: RoutePermission,
signal: AbortSignal
): Response {
if (!response.body) return response;
const reader = response.body.getReader();
let timer: ReturnType<typeof setInterval> | undefined;
let closed = false;
let closeStream: () => void = () => {};
const cleanup = () => { if (timer) clearInterval(timer); signal.removeEventListener('abort', closeStream); };
const cleanup = () => {
if (timer) clearInterval(timer);
signal.removeEventListener('abort', closeStream);
};
const stream = new ReadableStream<Uint8Array>({
start(controller) {
closeStream = () => {
if (closed) return; closed = true; cleanup();
void reader.cancel().catch(() => {}); controller.close();
if (closed) return;
closed = true;
cleanup();
void reader.cancel().catch(() => {});
controller.close();
};
timer = setInterval(() => {
try {
const current = readSessionHash(tokenHash);
if (!current || current.user.mustChangePassword || !roleAllows(current.user.role,permission)) closeStream();
} catch { closeStream(); }
},15_000);
signal.addEventListener('abort',closeStream,{once:true});
if (signal.aborted) { closeStream(); return; }
if (
!current ||
current.user.mustChangePassword ||
!roleAllows(current.user.role, permission)
)
closeStream();
} catch {
closeStream();
}
}, 15_000);
signal.addEventListener('abort', closeStream, { once: true });
if (signal.aborted) {
closeStream();
return;
}
},
async pull(controller) {
try {
const next = await reader.read();
if (closed) return;
if (next.done) { closed=true; cleanup(); controller.close(); }
else controller.enqueue(next.value);
} catch (cause) { if (!closed) { closed=true; cleanup(); controller.error(cause); } }
if (next.done) {
closed = true;
cleanup();
controller.close();
} else controller.enqueue(next.value);
} catch (cause) {
if (!closed) {
closed = true;
cleanup();
controller.error(cause);
}
}
},
cancel() { closed=true; cleanup(); return reader.cancel(); }
cancel() {
closed = true;
cleanup();
return reader.cancel();
}
});
return new Response(stream, {
status: response.status,
statusText: response.statusText,
headers: response.headers
});
return new Response(stream,{status:response.status,statusText:response.statusText,headers:response.headers});
}
export const authenticationHandle: Handle = async ({ event, resolve }) => {
event.locals.user = null; event.locals.sessionHash = null;
event.locals.user = null;
event.locals.sessionHash = null;
const routeId = event.route.id;
const method = event.request.method.toUpperCase();
if (publicAsset(routeId,event.url.pathname,method)) return resolve(event);
const permission = routePermission(routeId,method);
if (!routeId?.startsWith('/api/agent/') && /^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization')??'')) {
return json({error:'Host credentials only authorize the host agent API.'},{status:403});
if (publicAsset(routeId, event.url.pathname, method)) return resolve(event);
const permission = routePermission(routeId, method);
if (
!routeId?.startsWith('/api/agent/') &&
/^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization') ?? '')
) {
return json(
{ error: 'Host credentials only authorize the host agent API.' },
{ status: 403 }
);
}
if (permission === 'machine') return resolve(event);
const api = routeId?.startsWith('/api/') || event.url.pathname.startsWith('/api/');
try {
requireUserTransport(event.url);
if (!['GET','HEAD','OPTIONS'].includes(method)) requireSameOrigin(event.request,event.url);
} catch (cause) { if (api) return authError(cause); throw cause; }
if (!['GET', 'HEAD', 'OPTIONS'].includes(method))
requireSameOrigin(event.request, event.url);
} catch (cause) {
if (api) return authError(cause);
throw cause;
}
const session = readSession(event.cookies.get(sessionCookie(event.url)));
event.locals.user = session?.user??null;
event.locals.sessionHash = session?.tokenHash??null;
event.locals.user = session?.user ?? null;
event.locals.sessionHash = session?.tokenHash ?? null;
const setup = setupRequired();
if (permission !== 'public') {
if (!session) {
if (api) return json({error:setup?'Complete administrator setup before using the management API.':'Sign in to continue.',setupRequired:setup},{status:401,headers:{'Cache-Control':'no-store'}});
redirect(303,setup?'/setup':`/login?next=${encodeURIComponent(event.url.pathname+event.url.search)}`);
if (api)
return json(
{
error: setup
? 'Complete administrator setup before using the management API.'
: 'Sign in to continue.',
setupRequired: setup
},
{ status: 401, headers: { 'Cache-Control': 'no-store' } }
);
redirect(
303,
setup
? '/setup'
: `/login?next=${encodeURIComponent(event.url.pathname + event.url.search)}`
);
}
if (session.user.mustChangePassword && permission !== 'authenticated') {
if (api) return json({error:'Change your temporary password from Account before continuing.',mustChangePassword:true},{status:403,headers:{'Cache-Control':'no-store'}});
redirect(303,'/account');
if (api)
return json(
{
error: 'Change your temporary password from Account before continuing.',
mustChangePassword: true
},
{ status: 403, headers: { 'Cache-Control': 'no-store' } }
);
redirect(303, '/account');
}
if (!roleAllows(session.user.role,permission)) {
const message = permission==='admin'?'Administrator access is required.':'Operator access is required for this action.';
if (api) return json({error:message},{status:403,headers:{'Cache-Control':'no-store'}});
error(403,message);
if (!roleAllows(session.user.role, permission)) {
const message =
permission === 'admin'
? 'Administrator access is required.'
: 'Operator access is required for this action.';
if (api)
return json(
{ error: message },
{ status: 403, headers: { 'Cache-Control': 'no-store' } }
);
error(403, message);
}
// Launch APIs can also save credentials. Operators may launch with saved
// keys, but only administrators may introduce or replace server keys.
if (session.user.role === 'operator' && method === 'POST' && ['/api/servers','/api/deployment-collections'].includes(routeId??'')) {
if (
session.user.role === 'operator' &&
method === 'POST' &&
['/api/servers', '/api/deployment-collections'].includes(routeId ?? '')
) {
try {
const body = await authBody(event.request.clone(),1024*1024);
if (body.plutoKey || (body.keys && typeof body.keys==='object' && Object.values(body.keys).some(Boolean))) {
return json({error:'An administrator must save server keys before an operator launches these servers.'},{status:403});
const body = await authBody(event.request.clone(), 1024 * 1024);
if (
body.plutoKey ||
(body.keys &&
typeof body.keys === 'object' &&
Object.values(body.keys).some(Boolean))
) {
return json(
{
error: 'An administrator must save server keys before an operator launches these servers.'
},
{ status: 403 }
);
}
} catch (cause) { return authError(cause); }
} catch (cause) {
return authError(cause);
}
}
} else if (['/login','/setup'].includes(routeId??'')) {
if (session) redirect(303,session.user.mustChangePassword?'/account':'/');
if (routeId==='/setup'&&!setup) redirect(303,'/login');
if (routeId==='/login'&&setup) redirect(303,'/setup');
} else if (['/login', '/setup'].includes(routeId ?? '')) {
if (session) redirect(303, session.user.mustChangePassword ? '/account' : '/');
if (routeId === '/setup' && !setup) redirect(303, '/login');
if (routeId === '/login' && setup) redirect(303, '/setup');
}
let response = await withRequestUsername(session?.user.username ?? null, () => resolve(event));
if (session && !['GET','HEAD','OPTIONS'].includes(method) && response.status < 400) {
recordAudit(session.user.id,'request',routeId,{method,status:response.status,path:event.url.pathname});
if (session && !['GET', 'HEAD', 'OPTIONS'].includes(method) && response.status < 400) {
recordAudit(session.user.id, 'request', routeId, {
method,
status: response.status,
path: event.url.pathname
});
}
if (session && response.headers.get('content-type')?.startsWith('text/event-stream')) {
response = protectEventStream(response,session.tokenHash,permission,event.request.signal);
response = protectEventStream(
response,
session.tokenHash,
permission,
event.request.signal
);
}
const headers = new Headers(response.headers);
headers.set('Cache-Control','no-store');
headers.set('Vary',[headers.get('Vary'),'Cookie'].filter(Boolean).join(', '));
headers.set('Referrer-Policy','same-origin');
headers.set('X-Content-Type-Options','nosniff');
headers.set('X-Frame-Options','DENY');
return new Response(response.body,{status:response.status,statusText:response.statusText,headers});
headers.set('Cache-Control', 'no-store');
headers.set('Vary', [headers.get('Vary'), 'Cookie'].filter(Boolean).join(', '));
headers.set('Referrer-Policy', 'same-origin');
headers.set('X-Content-Type-Options', 'nosniff');
headers.set('X-Frame-Options', 'DENY');
return new Response(response.body, {
status: response.status,
statusText: response.statusText,
headers
});
};
src/lib/server/auth/http.ts
+76 -22
View File
@@ -2,48 +2,102 @@ import { error, isHttpError, json, type Cookies } from '@sveltejs/kit';
import { AuthError, SESSION_COOKIE, type AuthResult } from './index.js';
import type { AuthUser } from '../../auth.js';
export function sessionCookie(url: URL): string { return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev'; }
export function sessionCookie(url: URL): string {
return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev';
}
export function requireUserTransport(url: URL): void {
if (url.protocol === 'https:') return;
if (process.env.EC_AUTH_ALLOW_HTTP === 'true' && ['localhost','127.0.0.1','[::1]','::1'].includes(url.hostname)) return;
error(426, 'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.');
if (
process.env.EC_AUTH_ALLOW_HTTP === 'true' &&
['localhost', '127.0.0.1', '[::1]', '::1'].includes(url.hostname)
)
return;
error(
426,
'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.'
);
}
export function requireSameOrigin(request: Request, url: URL): void {
if (request.headers.get('origin') !== url.origin || request.headers.get('sec-fetch-site') === 'cross-site') {
if (
request.headers.get('origin') !== url.origin ||
request.headers.get('sec-fetch-site') === 'cross-site'
) {
error(403, 'This action requires a same-origin request.');
}
}
export function sessionResponse(result: AuthResult, cookies: Cookies, url: URL) {
cookies.set(sessionCookie(url), result.session.token, { path: '/', httpOnly: true, secure: url.protocol === 'https:',
sameSite: 'lax', expires: new Date(result.session.expiresAt) });
return json({ user: result.user }, { headers: { 'Cache-Control':'no-store' } });
cookies.set(sessionCookie(url), result.session.token, {
path: '/',
httpOnly: true,
secure: url.protocol === 'https:',
sameSite: 'lax',
expires: new Date(result.session.expiresAt)
});
return json({ user: result.user }, { headers: { 'Cache-Control': 'no-store' } });
}
export function requireUser(user: AuthUser | null): AuthUser {
if (!user) throw new AuthError('Sign in to continue.',401);
if (!user) throw new AuthError('Sign in to continue.', 401);
return user;
}
export async function authBody(request: Request, maxBytes = 8192): Promise<Record<string, unknown>> {
if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json')) throw new AuthError('Send a JSON request.',415);
if (Number(request.headers.get('content-length')) > maxBytes) throw new AuthError('Request is too large.',413);
const reader = request.body?.getReader(); if (!reader) throw new AuthError('A JSON object is required.');
const chunks: Uint8Array[] = []; let length = 0;
export async function authBody(
request: Request,
maxBytes = 8192
): Promise<Record<string, unknown>> {
if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json'))
throw new AuthError('Send a JSON request.', 415);
if (Number(request.headers.get('content-length')) > maxBytes)
throw new AuthError('Request is too large.', 413);
const reader = request.body?.getReader();
if (!reader) throw new AuthError('A JSON object is required.');
const chunks: Uint8Array[] = [];
let length = 0;
try {
for (;;) {
const { done, value } = await reader.read(); if (done) break;
const { done, value } = await reader.read();
if (done) break;
length += value.byteLength;
if (length > maxBytes) { void reader.cancel().catch(() => {}); throw new AuthError('Request is too large.',413); }
if (length > maxBytes) {
void reader.cancel().catch(() => {});
throw new AuthError('Request is too large.', 413);
}
chunks.push(value);
}
} finally { reader.releaseLock(); }
} finally {
reader.releaseLock();
}
let body: unknown;
try { body = JSON.parse(Buffer.concat(chunks,length).toString('utf8')); } catch { throw new AuthError('Invalid JSON.'); }
if (!body || typeof body !== 'object' || Array.isArray(body)) throw new AuthError('A JSON object is required.');
try {
body = JSON.parse(Buffer.concat(chunks, length).toString('utf8'));
} catch {
throw new AuthError('Invalid JSON.');
}
if (!body || typeof body !== 'object' || Array.isArray(body))
throw new AuthError('A JSON object is required.');
return body as Record<string, unknown>;
}
export function authError(cause: unknown): Response {
if (isHttpError(cause)) return json({ error: cause.body.message }, { status:cause.status, headers:{'Cache-Control':'no-store'} });
if (isHttpError(cause))
return json(
{ error: cause.body.message },
{ status: cause.status, headers: { 'Cache-Control': 'no-store' } }
);
const message = cause instanceof Error ? cause.message : 'Authentication request failed.';
const status = cause instanceof AuthError ? cause.status : message.includes('busy') ? 429 : message.startsWith('Use a password') ? 400 : 500;
return json({ error: status === 500 ? 'The authentication request could not be completed.' : message },
{ status, headers:{'Cache-Control':'no-store',...(status===429?{'Retry-After':'900'}:{})} });
const status =
cause instanceof AuthError
? cause.status
: message.includes('busy')
? 429
: message.startsWith('Use a password')
? 400
: 500;
return json(
{ error: status === 500 ? 'The authentication request could not be completed.' : message },
{
status,
headers: {
'Cache-Control': 'no-store',
...(status === 429 ? { 'Retry-After': '900' } : {})
}
}
);
}
src/lib/server/auth/index.ts
+342 -106
View File
@@ -8,179 +8,415 @@ export const SESSION_COOKIE = '__Host-ec_session';
export const SESSION_LIFETIME_MS = 12 * 60 * 60_000;
export const SESSION_IDLE_MS = 30 * 60_000;
const digest = (value: string) => createHash('sha256').update(value).digest('hex');
function database() { const handle = db(); ensureAuthSchema(handle); return handle; }
type UserRow = { id: string; username: string; display_name: string; password_hash: string; role: UserRole; disabled: number;
must_change_password: number; created_at: number; updated_at: number; last_login_at: number | null };
function database() {
const handle = db();
ensureAuthSchema(handle);
return handle;
}
type UserRow = {
id: string;
username: string;
display_name: string;
password_hash: string;
role: UserRole;
disabled: number;
must_change_password: number;
created_at: number;
updated_at: number;
last_login_at: number | null;
};
export class AuthError extends Error {
constructor(message: string, public status = 400) { super(message); }
constructor(
message: string,
public status = 400
) {
super(message);
}
}
function present(row: UserRow): AuthUser {
return { id: row.id, username: row.username, displayName: row.display_name, role: row.role, disabled: !!row.disabled,
mustChangePassword: !!row.must_change_password, createdAt: new Date(row.created_at).toISOString(),
updatedAt: new Date(row.updated_at).toISOString(), lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString() };
return {
id: row.id,
username: row.username,
displayName: row.display_name,
role: row.role,
disabled: !!row.disabled,
mustChangePassword: !!row.must_change_password,
createdAt: new Date(row.created_at).toISOString(),
updatedAt: new Date(row.updated_at).toISOString(),
lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString()
};
}
function userRow(id: string) {
return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined;
}
export function getUser(id: string): AuthUser | null {
const row = userRow(id);
return row ? present(row) : null;
}
export function listUsers(): AuthUser[] {
return (
database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[]
).map(present);
}
export function setupRequired(): boolean {
return !database().prepare('SELECT 1 FROM auth_users LIMIT 1').get();
}
function userRow(id: string) { return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined; }
export function getUser(id: string): AuthUser | null { const row = userRow(id); return row ? present(row) : null; }
export function listUsers(): AuthUser[] { return (database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[]).map(present); }
export function setupRequired(): boolean { return !(database().prepare('SELECT 1 FROM auth_users LIMIT 1').get()); }
function username(value: unknown): string {
if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value)) throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.');
if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value))
throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.');
const normalized = value.toLowerCase();
if (normalized === systemUsername.toLowerCase()) throw new AuthError(`${systemUsername} is reserved for system actions. Choose another username.`);
if (normalized === systemUsername.toLowerCase())
throw new AuthError(
`${systemUsername} is reserved for system actions. Choose another username.`
);
return normalized;
}
function displayName(value: unknown, fallback: string): string {
const name = value === undefined ? fallback : value;
if (typeof name !== 'string' || !name.trim() || name.length > 100 || /[\x00-\x1f\x7f]/.test(name)) throw new AuthError('Display name must contain 1–100 printable characters.');
if (
typeof name !== 'string' ||
!name.trim() ||
name.length > 100 ||
// eslint-disable-next-line no-control-regex -- Reject control characters in display names.
/[\x00-\x1f\x7f]/.test(name)
)
throw new AuthError('Display name must contain 1–100 printable characters.');
return name.trim();
}
function role(value: unknown): UserRole {
if (!['admin','operator','viewer'].includes(value as string)) throw new AuthError('Choose administrator, operator or viewer.');
if (!['admin', 'operator', 'viewer'].includes(value as string))
throw new AuthError('Choose administrator, operator or viewer.');
return value as UserRole;
}
export function recordAudit(actorId: string | null, action: string, targetId: string | null = null, details: Record<string, unknown> = {}): void {
database().prepare('INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)')
export function recordAudit(
actorId: string | null,
action: string,
targetId: string | null = null,
details: Record<string, unknown> = {}
): void {
database()
.prepare(
'INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)'
)
.run(actorId, action, targetId, JSON.stringify(details), Date.now());
}
export function listAuthAudit() {
return database().prepare(`SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt
FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200`).all();
return database()
.prepare(
`SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt
FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200`
)
.all();
}
/** Persistent fixed windows are consumed before hashing, including failed setup. */
export function consumeAuthAttempt(clientAddress: string, account: string, kind = 'login'): void {
const now = Date.now(); const windowMs = 15 * 60_000;
const now = Date.now();
const windowMs = 15 * 60_000;
const ipKey = digest(`${kind}:ip:${clientAddress}`);
const accountKey = digest(`${kind}:account:${account.toLowerCase()}`);
database().transaction(() => {
database().prepare('DELETE FROM auth_rate_limits WHERE window_start < ?').run(now - 24 * 60 * 60_000);
for (const [key, limit] of [[ipKey, 30], [accountKey, 10]] as const) {
const previous = database().prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?').get(key) as { window_start: number; attempts: number } | undefined;
if (previous && previous.window_start > now - windowMs && previous.attempts >= limit) throw new AuthError('Too many attempts. Try again in 15 minutes.', 429);
}
for (const key of [ipKey, accountKey]) database().prepare(`INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1)
database()
.transaction(() => {
database()
.prepare('DELETE FROM auth_rate_limits WHERE window_start < ?')
.run(now - 24 * 60 * 60_000);
for (const [key, limit] of [
[ipKey, 30],
[accountKey, 10]
] as const) {
const previous = database()
.prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?')
.get(key) as { window_start: number; attempts: number } | undefined;
if (
previous &&
previous.window_start > now - windowMs &&
previous.attempts >= limit
)
throw new AuthError('Too many attempts. Try again in 15 minutes.', 429);
}
for (const key of [ipKey, accountKey])
database()
.prepare(
`INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1)
ON CONFLICT(key) DO UPDATE SET attempts=CASE WHEN window_start <= ? THEN 1 ELSE attempts+1 END,
window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END`).run(key, now, now - windowMs, now - windowMs);
}).immediate();
window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END`
)
.run(key, now, now - windowMs, now - windowMs);
})
.immediate();
}
function insertUser(input: { username: string; displayName: string; role: UserRole; passwordHash: string; mustChangePassword: boolean }): AuthUser {
if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username)) throw new AuthError('That username is already in use.', 409);
const id = randomUUID(); const now = Date.now();
database().prepare(`INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`)
.run(id,input.username,input.displayName,input.passwordHash,input.role,Number(input.mustChangePassword),now,now);
function insertUser(input: {
username: string;
displayName: string;
role: UserRole;
passwordHash: string;
mustChangePassword: boolean;
}): AuthUser {
if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username))
throw new AuthError('That username is already in use.', 409);
const id = randomUUID();
const now = Date.now();
database()
.prepare(
`INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`
)
.run(
id,
input.username,
input.displayName,
input.passwordHash,
input.role,
Number(input.mustChangePassword),
now,
now
);
return getUser(id)!;
}
export interface AuthResult { user: AuthUser; session: { token: string; expiresAt: number } }
export async function bootstrapAdmin(input: Record<string, unknown>, clientAddress: string): Promise<AuthResult> {
export interface AuthResult {
user: AuthUser;
session: { token: string; expiresAt: number };
}
export async function bootstrapAdmin(
input: Record<string, unknown>,
clientAddress: string
): Promise<AuthResult> {
consumeAuthAttempt(clientAddress, 'bootstrap', 'setup');
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
const expected = process.env.EC_SETUP_TOKEN;
if (!expected || expected.length < 32) throw new AuthError('The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.', 503);
if (typeof input.token !== 'string' || !timingSafeEqual(Buffer.from(digest(input.token), 'hex'), Buffer.from(digest(expected), 'hex'))) throw new AuthError('Invalid setup token.', 401);
const name = username(input.username); const label = displayName(input.displayName, name);
if (!expected || expected.length < 32)
throw new AuthError(
'The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.',
503
);
if (
typeof input.token !== 'string' ||
!timingSafeEqual(
Buffer.from(digest(input.token), 'hex'),
Buffer.from(digest(expected), 'hex')
)
)
throw new AuthError('Invalid setup token.', 401);
const name = username(input.username);
const label = displayName(input.displayName, name);
const passwordHash = await hashPassword(input.password);
return database().transaction(() => {
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
const user = insertUser({ username: name, displayName: label, passwordHash, role: 'admin', mustChangePassword: false });
recordAudit(user.id, 'setup', user.id);
return { user, session: createSession(user.id) };
}).immediate();
return database()
.transaction(() => {
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
const user = insertUser({
username: name,
displayName: label,
passwordHash,
role: 'admin',
mustChangePassword: false
});
recordAudit(user.id, 'setup', user.id);
return { user, session: createSession(user.id) };
})
.immediate();
}
export async function authenticatePassword(input: Record<string, unknown>, clientAddress: string): Promise<AuthResult> {
const name = typeof input.username === 'string' ? input.username.slice(0,64).toLowerCase() : '';
export async function authenticatePassword(
input: Record<string, unknown>,
clientAddress: string
): Promise<AuthResult> {
const name =
typeof input.username === 'string' ? input.username.slice(0, 64).toLowerCase() : '';
consumeAuthAttempt(clientAddress, name);
const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as UserRow | undefined;
const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as
UserRow | undefined;
const valid = await verifyPassword(input.password, row?.password_hash ?? null);
const result = database().transaction(() => {
const current = row ? userRow(row.id) : undefined;
if (!valid || !current || current.disabled || current.password_hash !== row?.password_hash) {
recordAudit(null, 'login.failed'); return null;
}
database().prepare('UPDATE auth_users SET last_login_at=? WHERE id=?').run(Date.now(), current.id);
recordAudit(current.id, 'login', current.id);
return { user: getUser(current.id)!, session: createSession(current.id) };
}).immediate();
const result = database()
.transaction(() => {
const current = row ? userRow(row.id) : undefined;
if (
!valid ||
!current ||
current.disabled ||
current.password_hash !== row?.password_hash
) {
recordAudit(null, 'login.failed');
return null;
}
database()
.prepare('UPDATE auth_users SET last_login_at=? WHERE id=?')
.run(Date.now(), current.id);
recordAudit(current.id, 'login', current.id);
return { user: getUser(current.id)!, session: createSession(current.id) };
})
.immediate();
if (!result) throw new AuthError('Invalid username or password.', 401);
return result;
}
export function createSession(userId: string): { token: string; expiresAt: number } {
const now = Date.now(); const user = userRow(userId);
const now = Date.now();
const user = userRow(userId);
if (!user || user.disabled) throw new AuthError('Account is unavailable.', 401);
const token = randomBytes(32).toString('base64url'); const expiresAt = now + SESSION_LIFETIME_MS;
database().prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?').run(now,now-SESSION_IDLE_MS);
database().prepare('INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)').run(digest(token),userId,now,expiresAt,now);
const token = randomBytes(32).toString('base64url');
const expiresAt = now + SESSION_LIFETIME_MS;
database()
.prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?')
.run(now, now - SESSION_IDLE_MS);
database()
.prepare(
'INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)'
)
.run(digest(token), userId, now, expiresAt, now);
return { token, expiresAt };
}
export function readSession(token: string | undefined, touch = true): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
export function readSession(
token: string | undefined,
touch = true
): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
if (!token || !/^[A-Za-z0-9_-]{43}$/.test(token)) return null;
return readSessionHash(digest(token), touch);
}
export function readSessionHash(tokenHash: string, touch = false): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
export function readSessionHash(
tokenHash: string,
touch = false
): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
const now = Date.now();
const row = database().prepare(`SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id
WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0`).get(tokenHash,now,now-SESSION_IDLE_MS) as (UserRow & { expires_at: number; last_seen_at: number }) | undefined;
const row = database()
.prepare(
`SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id
WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0`
)
.get(tokenHash, now, now - SESSION_IDLE_MS) as
(UserRow & { expires_at: number; last_seen_at: number }) | undefined;
if (!row) return null;
if (touch && row.last_seen_at < now-60_000) database().prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?').run(now,tokenHash);
if (touch && row.last_seen_at < now - 60_000)
database()
.prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?')
.run(now, tokenHash);
return { user: present(row), tokenHash, expiresAt: row.expires_at };
}
export function revokeSession(token: string | undefined): void { if (token) database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token)); }
export function revokeSession(token: string | undefined): void {
if (token)
database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token));
}
export function revokeUserSessions(actorId: string, userId: string): void {
database().transaction(() => {
requireAdministrator(actorId);
if (!userRow(userId)) throw new AuthError('User not found.',404);
if (!userRow(userId)) throw new AuthError('User not found.', 404);
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
recordAudit(actorId,'sessions.revoked',userId);
recordAudit(actorId, 'sessions.revoked', userId);
})();
}
function requireAdministrator(actorId: string) {
const actor = userRow(actorId);
if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password) throw new AuthError('Administrator access is required.',403);
if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password)
throw new AuthError('Administrator access is required.', 403);
}
export async function createUser(actorId: string, input: Record<string, unknown>): Promise<AuthUser> {
export async function createUser(
actorId: string,
input: Record<string, unknown>
): Promise<AuthUser> {
requireAdministrator(actorId);
const name = username(input.username), label = displayName(input.displayName,name), userRole = role(input.role);
const name = username(input.username),
label = displayName(input.displayName, name),
userRole = role(input.role);
const passwordHash = await hashPassword(input.password);
return database().transaction(() => {
requireAdministrator(actorId);
const user = insertUser({ username:name,displayName:label,role:userRole,passwordHash,mustChangePassword:true });
recordAudit(actorId,'user.created',user.id,{role:userRole}); return user;
}).immediate();
return database()
.transaction(() => {
requireAdministrator(actorId);
const user = insertUser({
username: name,
displayName: label,
role: userRole,
passwordHash,
mustChangePassword: true
});
recordAudit(actorId, 'user.created', user.id, { role: userRole });
return user;
})
.immediate();
}
export async function updateUser(actorId: string, id: string, input: Record<string, unknown>): Promise<AuthUser> {
export async function updateUser(
actorId: string,
id: string,
input: Record<string, unknown>
): Promise<AuthUser> {
requireAdministrator(actorId);
if (input.disabled !== undefined && typeof input.disabled !== 'boolean') throw new AuthError('Disabled must be true or false.');
if (input.password !== undefined && actorId === id) throw new AuthError('Change your own password from Account using your current password.');
if (input.disabled !== undefined && typeof input.disabled !== 'boolean')
throw new AuthError('Disabled must be true or false.');
if (input.password !== undefined && actorId === id)
throw new AuthError('Change your own password from Account using your current password.');
const passwordHash = input.password === undefined ? null : await hashPassword(input.password);
return database().transaction(() => {
requireAdministrator(actorId);
const current = userRow(id); if (!current) throw new AuthError('User not found.',404);
const nextRole = input.role === undefined ? current.role : role(input.role);
const disabled = input.disabled === undefined ? current.disabled : Number(input.disabled);
const label = displayName(input.displayName,current.display_name);
if (current.role === 'admin' && !current.disabled && (nextRole !== 'admin' || disabled)) {
const other = database().prepare("SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1").get(id);
if (!other) throw new AuthError('Keep at least one enabled administrator.',409);
}
database().prepare(`UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?`)
.run(label,nextRole,disabled,passwordHash??current.password_hash,passwordHash?1:current.must_change_password,Date.now(),id);
if (passwordHash || disabled !== current.disabled || nextRole !== current.role) database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id);
recordAudit(actorId,'user.updated',id,{role:nextRole,disabled:!!disabled,passwordReset:!!passwordHash});
return getUser(id)!;
}).immediate();
return database()
.transaction(() => {
requireAdministrator(actorId);
const current = userRow(id);
if (!current) throw new AuthError('User not found.', 404);
const nextRole = input.role === undefined ? current.role : role(input.role);
const disabled =
input.disabled === undefined ? current.disabled : Number(input.disabled);
const label = displayName(input.displayName, current.display_name);
if (
current.role === 'admin' &&
!current.disabled &&
(nextRole !== 'admin' || disabled)
) {
const other = database()
.prepare(
"SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1"
)
.get(id);
if (!other) throw new AuthError('Keep at least one enabled administrator.', 409);
}
database()
.prepare(
`UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?`
)
.run(
label,
nextRole,
disabled,
passwordHash ?? current.password_hash,
passwordHash ? 1 : current.must_change_password,
Date.now(),
id
);
if (passwordHash || disabled !== current.disabled || nextRole !== current.role)
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id);
recordAudit(actorId, 'user.updated', id, {
role: nextRole,
disabled: !!disabled,
passwordReset: !!passwordHash
});
return getUser(id)!;
})
.immediate();
}
export async function changePassword(userId: string, currentPassword: unknown, newPassword: unknown): Promise<AuthResult> {
export async function changePassword(
userId: string,
currentPassword: unknown,
newPassword: unknown
): Promise<AuthResult> {
const original = userRow(userId);
if (!original || original.disabled || !await verifyPassword(currentPassword,original.password_hash)) throw new AuthError('Current password is incorrect.',400);
if (
!original ||
original.disabled ||
!(await verifyPassword(currentPassword, original.password_hash))
)
throw new AuthError('Current password is incorrect.', 400);
if (currentPassword === newPassword) throw new AuthError('Choose a different password.');
const passwordHash = await hashPassword(newPassword);
return database().transaction(() => {
const current = userRow(userId);
if (!current || current.disabled || current.password_hash !== original.password_hash) throw new AuthError('Account changed during password verification. Sign in again.',409);
database().prepare('UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?').run(passwordHash,Date.now(),userId);
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
recordAudit(userId,'password.changed',userId);
return { user: getUser(userId)!, session: createSession(userId) };
}).immediate();
return database()
.transaction(() => {
const current = userRow(userId);
if (!current || current.disabled || current.password_hash !== original.password_hash)
throw new AuthError(
'Account changed during password verification. Sign in again.',
409
);
database()
.prepare(
'UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?'
)
.run(passwordHash, Date.now(), userId);
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
recordAudit(userId, 'password.changed', userId);
return { user: getUser(userId)!, session: createSession(userId) };
})
.immediate();
}
src/lib/server/auth/passwords.js
+15 -5
View File
@@ -15,12 +15,14 @@ export function validatePassword(password) {
/** @param {string} password @param {string} salt @returns {Promise<Buffer>} */
function derive(password, salt) {
if (active >= 2) return Promise.reject(new Error('Password service is busy. Try again shortly.'));
if (active >= 2)
return Promise.reject(new Error('Password service is busy. Try again shortly.'));
active++;
return new Promise((resolve, reject) => {
scrypt(password, salt, 64, parameters, (error, result) => {
active--;
if (error) reject(error); else resolve(result);
if (error) reject(error);
else resolve(result);
});
});
}
@@ -37,9 +39,17 @@ export async function hashPassword(password) {
export async function verifyPassword(password, encoded) {
if (typeof password !== 'string' || [...password].length > 128) return false;
const parts = encoded?.split('$');
const valid = parts?.length === 6 && parts[0] === 'scrypt' && parts[1] === '131072' && parts[2] === '8' && parts[3] === '1'
&& /^[a-f0-9]{64}$/.test(parts[4]) && /^[a-f0-9]{128}$/.test(parts[5]);
const valid =
parts?.length === 6 &&
parts[0] === 'scrypt' &&
parts[1] === '131072' &&
parts[2] === '8' &&
parts[3] === '1' &&
/^[a-f0-9]{64}$/.test(parts[4]) &&
/^[a-f0-9]{128}$/.test(parts[5]);
// Missing accounts perform the same expensive operation as real accounts.
const result = await derive(password, valid ? parts[4] : '0'.repeat(64));
return timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid;
return (
timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid
);
}
src/lib/server/auth/request-context.ts
+9 -3
View File
@@ -3,11 +3,17 @@ import { AsyncLocalStorage } from 'node:async_hooks';
const requester = new AsyncLocalStorage<{ username: string | null; active: boolean }>();
/** Use only the authenticated session, and stop attribution when the request finishes. */
export async function withRequestUsername<T>(username: string | null, work: () => T | Promise<T>): Promise<T> {
export async function withRequestUsername<T>(
username: string | null,
work: () => T | Promise<T>
): Promise<T> {
const context = { username, active: true };
return requester.run(context, async () => {
try { return await work(); }
finally { context.active = false; }
try {
return await work();
} finally {
context.active = false;
}
});
}
src/lib/server/auth/telemetry.ts
+17 -8
View File
@@ -1,24 +1,32 @@
import { createHash, timingSafeEqual } from 'node:crypto';
export type TelemetryAuthentication =
| { ok: true }
| { ok: false; status: 401 | 503; error: string };
{ ok: true } | { ok: false; status: 401 | 503; error: string };
const tokenPattern = /^[A-Za-z0-9._~+\/-]+=*$/;
const validToken = (value: string): boolean => value.length >= 32 && value.length <= 512 && tokenPattern.test(value);
const tokenPattern = /^[A-Za-z0-9._~+/-]+=*$/;
const validToken = (value: string): boolean =>
value.length >= 32 && value.length <= 512 && tokenPattern.test(value);
const hash = (value: string): Buffer => createHash('sha256').update(value, 'utf8').digest();
/** Machine telemetry has its own credential; browser sessions never authorize it. */
export function authenticateTelemetry(request: Request): TelemetryAuthentication {
const expected = process.env.EC_TELEMETRY_TOKEN ?? '';
if (!validToken(expected)) {
return { ok: false, status: 503, error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.' };
return {
ok: false,
status: 503,
error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.'
};
}
const authorization = request.headers.get('authorization');
const custom = request.headers.get('x-ec-telemetry-token');
const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+\/-]+=*)$/i)?.[1];
const denied = (): TelemetryAuthentication => ({ ok: false, status: 401, error: 'A valid game telemetry token is required.' });
const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+/-]+=*)$/i)?.[1];
const denied = (): TelemetryAuthentication => ({
ok: false,
status: 401,
error: 'A valid game telemetry token is required.'
});
// Reject malformed/ambiguous credentials rather than falling back to a
// second header. A shared secret must never come from a URL, cookie or body.
@@ -28,7 +36,8 @@ export function authenticateTelemetry(request: Request): TelemetryAuthentication
const supplied = bearer ?? custom;
if (!supplied) return denied();
const suppliedHash = hash(supplied);
if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom))) return denied();
if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom)))
return denied();
if (!timingSafeEqual(suppliedHash, hash(expected))) return denied();
return { ok: true };
}
src/lib/server/autoscaling/admission.ts
+55 -10
View File
@@ -4,29 +4,74 @@ import { scalingSnapshot } from './snapshot.js';
import { liveScaling, type ScalingPolicy, type ScalingDecision } from '../../autoscaling.js';
import { deploymentStage } from '../provisioning/history.js';
export interface ScalingIntent { policy_id: string; revision: number; decision: string; operation_id: string }
export interface ScalingIntent {
policy_id: string;
revision: number;
decision: string;
operation_id: string;
}
export function scalingIntent(operationId: string): ScalingIntent | undefined {
return db().prepare('SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0').get(operationId) as ScalingIntent | undefined;
return db()
.prepare(
'SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0'
)
.get(operationId) as ScalingIntent | undefined;
}
/** Rechecked before placement, after asynchronous preparation, and before dispatch. */
export function scalingAdmission(operationId: string): string | null {
const intent = scalingIntent(operationId);
if (!intent) return null;
const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(intent.policy_id) as { id: string; settings: string; mode: ScalingPolicy['mode']; revision: number; paused_reason: string | null; owns_map: number } | undefined;
const row = db()
.prepare('SELECT * FROM autoscaling_policies WHERE id=?')
.get(intent.policy_id) as
| {
id: string;
settings: string;
mode: ScalingPolicy['mode'];
revision: number;
paused_reason: string | null;
owns_map: number;
}
| undefined;
let reason: string | null = null;
const paused = (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
if (!row || !liveScaling(row.mode) || !row.owns_map || row.revision !== intent.revision || row.paused_reason || paused) reason = 'Autoscaling policy is paused or changed.';
const paused =
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1';
if (
!row ||
!liveScaling(row.mode) ||
!row.owns_map ||
row.revision !== intent.revision ||
row.paused_reason ||
paused
)
reason = 'Autoscaling policy is paused or changed.';
else {
const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, pausedReason: null, updatedAt: '' };
const policy: ScalingPolicy = {
...JSON.parse(row.settings),
id: row.id,
mode: row.mode,
revision: row.revision,
pausedReason: null,
updatedAt: ''
};
const requested = JSON.parse(intent.decision) as ScalingDecision;
const snapshot = scalingSnapshot(policy, Date.now(), operationId);
if (requested.action === 'create') {
snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server);
snapshot.blockers = snapshot.blockers.filter(b => b !== `Unresolved native instance ${requested.server}.`);
snapshot.reservedNames = snapshot.reservedNames.filter((n) => n !== requested.server);
snapshot.blockers = snapshot.blockers.filter(
(b) => b !== `Unresolved native instance ${requested.server}.`
);
}
const current = planScaling(policy, snapshot, Date.now());
if (current.action !== requested.action || current.server !== requested.server) reason = `Autoscaling intent no longer eligible: ${current.reason}`;
if (current.action !== requested.action || current.server !== requested.server)
reason = `Autoscaling intent no longer eligible: ${current.reason}`;
}
if (reason) deploymentStage(operationId, reason, { blocker: { code: 'autoscaling', message: reason, automatic: true } });
if (reason)
deploymentStage(operationId, reason, {
blocker: { code: 'autoscaling', message: reason, automatic: true }
});
return reason;
}
src/lib/server/autoscaling/authorization.ts
+77 -17
View File
@@ -7,46 +7,106 @@ export const autoscalingRemovalCapability = 'instance.autoscaling-remove';
export function removalAuthorization(operationId: string, hostId: string): object | null {
const intent = scalingIntent(operationId);
if (!intent) return null;
if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) throw new Error('Update the host agent before automatic removal.');
if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability))
throw new Error('Update the host agent before automatic removal.');
const reason = scalingAdmission(operationId);
if (reason) throw new Error(reason);
const expiresAt = Date.now() + 60_000;
db().prepare('INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)')
db()
.prepare(
'INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)'
)
.run(operationId, intent.policy_id, intent.revision, expiresAt);
return { policyId: intent.policy_id, revision: intent.revision, expiresAt };
}
/** Claim is the cancellation boundary. Lost replies never authorize a second claim. */
export function claimRemovalAuthorization(hostId: string, commandId: string, input: Record<string, unknown>): { authorized: boolean; validForMs?: number; reason?: string } {
export function claimRemovalAuthorization(
hostId: string,
commandId: string,
input: Record<string, unknown>
): { authorized: boolean; validForMs?: number; reason?: string } {
return db().transaction(() => {
const now = Date.now();
const row = db().prepare(`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
const row = db()
.prepare(
`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
JOIN native_provisioning n ON n.deployment_id=a.operation_id JOIN server_deployments d ON d.id=n.deployment_id
JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
JOIN ec_host_operations o ON o.id=n.operation_id AND o.host_id=i.host_id AND o.instance_id=i.id AND o.generation=i.generation
WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`).get(commandId, hostId) as
{ operation_id: string; policy_id: string; revision: number; expires_at: number; claimed_at: number | null; revoked_at: number | null; deployment_state: string; command_state: string } | undefined;
if (!row || input.policyId !== row.policy_id || input.revision !== row.revision || input.expiresAt !== row.expires_at
|| row.claimed_at !== null || row.revoked_at !== null || row.expires_at <= now
|| row.deployment_state !== 'running' || !['pending','accepted','running'].includes(row.command_state)) return { authorized: false, reason: 'Removal authorization expired, changed, or was already claimed.' };
WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`
)
.get(commandId, hostId) as
| {
operation_id: string;
policy_id: string;
revision: number;
expires_at: number;
claimed_at: number | null;
revoked_at: number | null;
deployment_state: string;
command_state: string;
}
| undefined;
if (
!row ||
input.policyId !== row.policy_id ||
input.revision !== row.revision ||
input.expiresAt !== row.expires_at ||
row.claimed_at !== null ||
row.revoked_at !== null ||
row.expires_at <= now ||
row.deployment_state !== 'running' ||
!['pending', 'accepted', 'running'].includes(row.command_state)
)
return {
authorized: false,
reason: 'Removal authorization expired, changed, or was already claimed.'
};
const reason = scalingAdmission(row.operation_id);
if (reason) { db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?').run(now, row.operation_id); return { authorized: false, reason }; }
db().prepare('UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL').run(now, row.operation_id);
if (reason) {
db()
.prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?')
.run(now, row.operation_id);
return { authorized: false, reason };
}
db()
.prepare(
'UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL'
)
.run(now, row.operation_id);
// Only a short final occupancy check may occur after this boundary.
return { authorized: true, validForMs: Math.min(5000, row.expires_at - now) };
})();
}
export function revokePolicyRemovals(policyId: string): void {
if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId)) return;
db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL').run(Date.now(), policyId);
if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId))
return;
db()
.prepare(
'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL'
)
.run(Date.now(), policyId);
const at = new Date().toISOString();
const commands = db().prepare(`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
const commands = db()
.prepare(
`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
JOIN autoscaling_authorizations a ON a.operation_id=n.deployment_id
WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`).all(policyId) as { id: string }[];
WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`
)
.all(policyId) as { id: string }[];
for (const command of commands) {
const result = JSON.stringify({ autoscalingInvalidated: true });
db().prepare("UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?").run(result, at, command.id);
db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')").run(command.id, at);
db()
.prepare(
"UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?"
)
.run(result, at, command.id);
db()
.prepare(
"INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')"
)
.run(command.id, at);
}
}
src/lib/server/autoscaling/index.ts
+428 -87
View File
@@ -1,6 +1,13 @@
import { createHash, randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingEvaluation, type ScalingReport, type ScalingView } from '../../autoscaling.js';
import {
liveScaling,
scalingSettings,
type ScalingPolicy,
type ScalingEvaluation,
type ScalingReport,
type ScalingView
} from '../../autoscaling.js';
import { emptyObservation, planScaling } from './planner.js';
import { parseServerName } from '../../map-aliases.js';
import { scalingSnapshot } from './snapshot.js';
@@ -14,33 +21,98 @@ import { getHost } from '../hosts/index.js';
import { scalingOwner } from './ownership.js';
import { getServer } from '../fleet/servers.js';
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; paused_reason: string | null; updated_at: string }
const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, pausedReason: row.paused_reason, updatedAt: row.updated_at });
export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); }
export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
export class ScalingError extends Error {
constructor(
message: string,
public status = 400
) {
super(message);
}
}
interface PolicyRow {
id: string;
revision: number;
settings: string;
mode: ScalingPolicy['mode'];
paused_reason: string | null;
updated_at: string;
}
const present = (row: PolicyRow): ScalingPolicy => ({
...JSON.parse(row.settings),
id: row.id,
revision: row.revision,
mode: row.mode,
pausedReason: row.paused_reason,
updatedAt: row.updated_at
});
export const listScalingPolicies = (): ScalingPolicy[] =>
(
db()
.prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug')
.all() as PolicyRow[]
).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined {
const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as
PolicyRow | undefined;
return row && present(row);
}
export const scalingGloballyPaused = (): boolean =>
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1';
export function evaluateScaling(input: unknown, id?: string, revision?: number): ScalingEvaluation {
const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined;
const settings = scalingSettings(input),
existing = id ? getScalingPolicy(id) : undefined;
if (id && !existing) throw new ScalingError('Policy not found.', 404);
if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'new-policy', revision: existing?.revision ?? 0,
pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
if (existing && existing.revision !== revision)
throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug))
throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = {
...settings,
id: existing?.id ?? 'new-policy',
revision: existing?.revision ?? 0,
pausedReason: null,
updatedAt: existing?.updatedAt ?? ''
};
const snapshot = scalingSnapshot(policy);
const blockers: string[] = [];
if (liveScaling(policy.mode)) {
const owner = scalingOwner(policy.game, policy.mapSlug);
if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`);
if (owner && owner !== policy.id)
blockers.push(
`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`
);
blockers.push(...snapshot.blockers);
for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`);
if (policy.mode === 'create-destroy') for (const member of snapshot.members) {
const hostId = getServer(member.name)?.host_id;
if (!hostId || !getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) blockers.push(`Update the host agent for ${member.name} before enabling automatic removal.`);
for (const member of snapshot.members)
if (!['occupied', 'empty'].includes(member.state))
blockers.push(
`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`
);
if (policy.mode === 'create-destroy')
for (const member of snapshot.members) {
const hostId = getServer(member.name)?.host_id;
if (
!hostId ||
!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)
)
blockers.push(
`Update the host agent for ${member.name} before enabling automatic removal.`
);
}
if (
db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(policy.id)
)
blockers.push('Settle the current autoscaling operation before editing a live policy.');
try {
nativeLaunchSelection({ automatic: true }, policy.game);
} catch (error) {
blockers.push((error as Error).message);
}
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.');
try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); }
}
const decision = planScaling(policy, snapshot, Date.now());
return { policy, snapshot, decision, blockers };
@@ -50,16 +122,48 @@ export function saveScaling(input: unknown, id?: string, revision?: number): Sca
return db().transaction(() => {
const evaluation = evaluateScaling(input, id, revision);
if (evaluation.blockers.length) throw new ScalingError(evaluation.blockers[0]);
const policy = evaluation.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
const policy = evaluation.policy,
settings = scalingSettings(policy),
nextId = id ?? randomUUID(),
at = new Date().toISOString();
if (id) revokePolicyRemovals(id);
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this scope.', 409);
db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
if (
!id &&
db()
.prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?')
.get(policy.game, policy.mapSlug)
)
throw new ScalingError('A policy already exists for this scope.', 409);
db()
.prepare(
`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`
)
.run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
// An edit starts a new observation interval; earlier inactivity is not authorization.
db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId);
db().prepare('UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?').run(liveScaling(policy.mode) ? 1 : 0, nextId);
if (liveScaling(policy.mode)) for (const member of evaluation.snapshot.members) db().prepare('INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)')
.run(nextId, member.name, member.identity, JSON.stringify({ identity: member.identity, observedAt: member.observedAt, emptySince: null, evaluatedAt: Date.now() }));
db()
.prepare(
'UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?'
)
.run(liveScaling(policy.mode) ? 1 : 0, nextId);
if (liveScaling(policy.mode))
for (const member of evaluation.snapshot.members)
db()
.prepare(
'INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)'
)
.run(
nextId,
member.name,
member.identity,
JSON.stringify({
identity: member.identity,
observedAt: member.observedAt,
emptySince: null,
evaluatedAt: Date.now()
})
);
reconcileScaling(getScalingPolicy(nextId)!);
return getScalingPolicy(nextId)!;
})();
@@ -69,10 +173,18 @@ export function deleteScaling(id: string, revision: number): void {
db().transaction(() => {
const policy = getScalingPolicy(id);
if (!policy) throw new ScalingError('Policy not found.', 404);
if (policy.revision !== revision) throw new ScalingError('Policy changed. Refresh before deleting.', 409);
if (policy.revision !== revision)
throw new ScalingError('Policy changed. Refresh before deleting.', 409);
// Keep admission evidence and ownership while a host command may still act.
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(id)) {
throw new ScalingError('Pause this policy and settle its outstanding Provisioning work before deleting it.', 409);
if (
db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(id)
) {
throw new ScalingError(
'Pause this policy and settle its outstanding Provisioning work before deleting it.',
409
);
}
revokePolicyRemovals(id);
db().prepare('DELETE FROM autoscaling_authorizations WHERE policy_id=?').run(id);
@@ -85,39 +197,105 @@ export function deleteScaling(id: string, revision: number): void {
export function setScalingGlobalPause(paused: boolean): void {
db().transaction(() => {
db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
if (paused) for (const policy of listScalingPolicies()) { revokePolicyRemovals(policy.id); reconcileScaling(policy); }
db()
.prepare(
"INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value"
)
.run(paused ? '1' : '0');
if (paused)
for (const policy of listScalingPolicies()) {
revokePolicyRemovals(policy.id);
reconcileScaling(policy);
}
})();
}
function reconcileScaling(policy: ScalingPolicy): void {
const intents = db().prepare(`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
const intents = db()
.prepare(
`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as
{ id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[];
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`
)
.all(policy.id) as {
id: string;
revision: number;
operation_id: string;
action: string;
state: string;
submitted_at: string | null;
command_id: string | null;
}[];
for (const intent of intents) {
if (intent.action === 'create' && ['queued','waiting','cancelled'].includes(intent.state) && !intent.command_id && !intent.submitted_at
&& (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
if (
intent.action === 'create' &&
['queued', 'waiting', 'cancelled'].includes(intent.state) &&
!intent.command_id &&
!intent.submitted_at &&
(scalingAdmission(intent.operation_id) || intent.state === 'cancelled')
)
cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
if (intent.action === 'destroy' && scalingAdmission(intent.operation_id)) {
revokePolicyRemovals(policy.id);
if (!intent.command_id) cancelRemoval(intent.operation_id, false);
}
const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null };
const terminal = ['done','warning','failed','cancelled'].includes(current.state);
const command = intent.command_id ? db().prepare('SELECT state,result FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string; result: string | null } | undefined : undefined;
const invalidated = command?.state === 'failed' && command.result && JSON.parse(command.result).autoscalingInvalidated === true;
const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed'
&& !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id);
const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending;
if (!invalidated && (current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) {
db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id);
const current = db()
.prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?')
.get(intent.operation_id) as { state: string; server_deleted_at: string | null };
const terminal = ['done', 'warning', 'failed', 'cancelled'].includes(current.state);
const command = intent.command_id
? (db()
.prepare('SELECT state,result FROM ec_host_operations WHERE id=?')
.get(intent.command_id) as { state: string; result: string | null } | undefined)
: undefined;
const invalidated =
command?.state === 'failed' &&
command.result &&
JSON.parse(command.result).autoscalingInvalidated === true;
const receiptPending =
current.state === 'cancelled' &&
!current.server_deleted_at &&
command?.state === 'completed' &&
!!db()
.prepare(
'SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL'
)
.get(intent.operation_id);
const settled =
(!intent.command_id ||
(!!command && ['completed', 'failed'].includes(command.state))) &&
!receiptPending;
if (
!invalidated &&
(current.state === 'failed' ||
current.state === 'interrupted' ||
command?.state === 'unknown') &&
liveScaling(policy.mode)
) {
db()
.prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?')
.run(
`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`,
new Date().toISOString(),
policy.id
);
}
if (terminal && settled) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id);
if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled')) db().prepare("UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?").run(policy.id);
if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled'))
db()
.prepare(
"UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?"
)
.run(policy.id);
}
}
if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) {
if (
!liveScaling(policy.mode) &&
!db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(policy.id)
) {
db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id);
}
}
@@ -131,48 +309,152 @@ export async function runAutoscaling(): Promise<boolean> {
if (policy.mode === 'off') continue;
db().transaction(() => {
const snapshot = scalingSnapshot(policy, now);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined;
if (scalingGloballyPaused())
snapshot.blockers.unshift('Autoscaling is globally paused.');
const pending = db()
.prepare(
`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`
)
.get(policy.id) as { id: string; stage: string } | undefined;
if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`);
const decision = planScaling(policy, snapshot, now);
for (const member of snapshot.members) {
// Keep the observation interval that justified a pending removal;
// admission reads fresh evidence and revokes it on any new blocker.
if (member.state === 'removing' && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0').get(policy.id, member.operationId)) continue;
const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now);
db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation));
if (
member.state === 'removing' &&
db()
.prepare(
'SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0'
)
.get(policy.id, member.operationId)
)
continue;
const observation = emptyObservation(
member,
{
identity: member.identity,
observedAt: member.observedAt,
emptySince: member.emptySince,
evaluatedAt: now
},
now
);
db()
.prepare(
`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`
)
.run(policy.id, member.name, member.identity, JSON.stringify(observation));
}
// Evidence includes provenance and timestamps but never keys or player rosters.
const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyPool: snapshot.keyPool, blockers: snapshot.blockers });
const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
const execute = liveScaling(policy.mode) && (decision.action === 'create' || decision.action === 'destroy');
const evidence = JSON.stringify({
members: snapshot.members,
reservations: snapshot.reservedNames,
keyPool: snapshot.keyPool,
blockers: snapshot.blockers
});
const fingerprint = createHash('sha256')
.update(
JSON.stringify([
policy.revision,
decision,
snapshot.members.map((m) => [m.name, m.identity, m.state])
])
)
.digest('hex');
const last = db()
.prepare(
'SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1'
)
.get(policy.id) as { id: string; fingerprint: string } | undefined;
const execute =
liveScaling(policy.mode) &&
(decision.action === 'create' || decision.action === 'destroy');
const decisionId = randomUUID();
if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)')
.run(decisionId, policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence);
if (!execute && last?.fingerprint === fingerprint)
db()
.prepare(
'UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?'
)
.run(now, evidence, last.id);
else
db()
.prepare(
'INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)'
)
.run(
decisionId,
policy.id,
policy.revision,
now,
now,
fingerprint,
JSON.stringify(decision),
evidence
);
if (execute) {
db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
db()
.prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?')
.run(decisionId);
try {
db().transaction(() => {
const result = decision.action === 'create'
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
const result =
decision.action === 'create'
? queueNativeCreationIntent({
server: decision.server!,
game: policy.game,
mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection(
{ automatic: true },
policy.game
),
createdBy: `autoscaling:${policy.id}`,
autoscalingDecisionId: decisionId
})
: destroyServer(
decision.server!,
`autoscaling:${policy.id}`,
decisionId
);
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
if (!result.deploymentId)
throw new Error('Provisioning did not return an operation identity.');
db()
.prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?')
.run(result.deploymentId, decisionId);
db()
.prepare(
"INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)"
)
.run(result.deploymentId, policy.id, decision.server);
submitted = true;
})();
} catch (error) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Scaling action could not be queued: ${(error as Error).message}`, policy.id);
db()
.prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?')
.run(decisionId);
db()
.prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?')
.run(
`Scaling action could not be queued: ${(error as Error).message}`,
policy.id
);
}
}
db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, decision.deficit);
db()
.prepare(
`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`
)
.run(
policy.id,
Math.floor(now / 15_000) * 15_000,
policy.mode,
decision.occupied,
decision.empty,
decision.unknown,
decision.deficit
);
})();
}
return submitted;
@@ -180,37 +462,96 @@ export async function runAutoscaling(): Promise<boolean> {
export function pruneAutoscaling(now = Date.now()): void {
const cutoff = now - 31 * 86400_000;
db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`).run(cutoff);
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1').run(cutoff);
db()
.prepare(
`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`
)
.run(cutoff);
db()
.prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1')
.run(cutoff);
db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
}
export function scalingReport(): ScalingReport {
const policies: ScalingView[] = listScalingPolicies().map(policy => {
const policies: ScalingView[] = listScalingPolicies().map((policy) => {
const snapshot = scalingSnapshot(policy);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const decision = planScaling(policy, snapshot, Date.now());
const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
const history = (
db()
.prepare(
`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[])
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state }));
const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled'];
const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map;
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`
)
.all(policy.id) as {
id: string;
evaluated_at: number;
last_seen_at: number;
decision: string;
operation_id: string | null;
operation_state: string | null;
}[]
).map((row) => ({
id: row.id,
at: row.evaluated_at,
lastSeenAt: row.last_seen_at,
decision: JSON.parse(row.decision),
operationId: row.operation_id,
operationState: row.operation_state
}));
const unsettled = db()
.prepare(
`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`
)
.all(policy.id) as ScalingView['unsettled'];
const ownsMap = !!(
db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as {
owns_map: number;
}
).owns_map;
// Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
const totals = db()
.prepare(
`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`
)
.get(policy.id) as ScalingView['totals'];
const operations = db()
.prepare(
`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed,
COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples,
AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds,
MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds
FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations'];
return { policy, snapshot, decision, blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap };
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`
)
.get(
policy.id,
new Date(Date.now() - 31 * 86400_000).toISOString()
) as ScalingView['operations'];
return {
policy,
snapshot,
decision,
blockers: [],
history,
lastEvaluatedAt: history[0]?.lastSeenAt ?? null,
totals,
operations,
unsettled,
ownsMap
};
});
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true };
return {
policies,
globalPaused: scalingGloballyPaused(),
liveAvailable: true,
removalAvailable: true
};
}
src/lib/server/autoscaling/ownership.ts
+36 -8
View File
@@ -4,18 +4,29 @@ import { liveScaling, type ScalingPolicy } from '../../autoscaling.js';
import { mapIdentity } from './snapshot.js';
function owningPolicy(game: string, mapSlug: string) {
return db().prepare("SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1")
.get(game, mapSlug, mapSlug) as { id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined;
return db()
.prepare(
"SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1"
)
.get(game, mapSlug, mapSlug) as
{ id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined;
}
/** Durable ownership prevents overlapping policies even while scaling is paused. */
export function scalingOwner(game: string, mapSlug: string): string | null {
return owningPolicy(game, mapSlug)?.id ?? null;
}
const ownershipMessage = (owner: string) => `Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`;
const ownershipMessage = (owner: string) =>
`Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`;
function manualLineupBlocker(game: string, mapSlug: string): string | null {
const owner = owningPolicy(game, mapSlug);
if (!owner || !liveScaling(owner.mode) || owner.paused_reason) return null;
if ((db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1') return null;
if (
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1'
)
return null;
return ownershipMessage(owner.id);
}
export function scalingOwnershipBlocker(server: string): string | null {
@@ -30,13 +41,30 @@ export function assertScalingOwnership(game: string, mapSlug: string, decisionId
}
const owner = scalingOwner(game, mapSlug);
if (!owner) throw new Error('Autoscaling no longer owns this map.');
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return;
if (
db()
.prepare(
'SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL'
)
.get(decisionId, owner)
)
return;
throw new Error(ownershipMessage(owner));
}
/** Operator cancellation is a policy pause, not a fresh scaling signal. */
export function pauseScalingForOperation(operationId: string): void {
const row = db().prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?').get(operationId) as { policy_id: string } | undefined;
const row = db()
.prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?')
.get(operationId) as { policy_id: string } | undefined;
if (!row) return;
db().prepare("UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?").run(new Date().toISOString(), row.policy_id);
db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL').run(Date.now(), row.policy_id);
db()
.prepare(
"UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?"
)
.run(new Date().toISOString(), row.policy_id);
db()
.prepare(
'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL'
)
.run(Date.now(), row.policy_id);
}
src/lib/server/autoscaling/planner.ts
+143 -45
View File
@@ -1,72 +1,170 @@
import type { ScalingDecision, ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import type {
ScalingDecision,
ScalingMember,
ScalingPolicy,
ScalingSnapshot
} from '../../autoscaling.js';
import { parseServerName, zombieMaps } from '../../map-aliases.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
export interface EmptyObservation { identity: string; observedAt: number | null; emptySince: number | null; evaluatedAt: number }
export interface EmptyObservation {
identity: string;
observedAt: number | null;
emptySince: number | null;
evaluatedAt: number;
}
/** Only distinct, continuous evidence can advance an empty interval. */
export function emptyObservation(member: ScalingMember, previous: EmptyObservation | undefined, now: number): EmptyObservation {
export function emptyObservation(
member: ScalingMember,
previous: EmptyObservation | undefined,
now: number
): EmptyObservation {
const stamp = member.observedAt;
const empty = member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
const continuous = empty && previous?.identity === member.identity && previous.emptySince !== null
&& (!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt)
&& previous.observedAt !== null && stamp >= previous.observedAt && stamp - previous.observedAt <= ONLINE_MAX_MS
&& now >= previous.evaluatedAt && now - previous.evaluatedAt <= ONLINE_MAX_MS;
return { identity: member.identity, observedAt: stamp, emptySince: empty ? continuous ? previous!.emptySince : stamp : null, evaluatedAt: now };
const empty =
member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
const continuous =
empty &&
previous?.identity === member.identity &&
previous.emptySince !== null &&
(!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt) &&
previous.observedAt !== null &&
stamp >= previous.observedAt &&
stamp - previous.observedAt <= ONLINE_MAX_MS &&
now >= previous.evaluatedAt &&
now - previous.evaluatedAt <= ONLINE_MAX_MS;
return {
identity: member.identity,
observedAt: stamp,
emptySince: empty ? (continuous ? previous!.emptySince : stamp) : null,
evaluatedAt: now
};
}
export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, now: number): ScalingDecision {
export function planScaling(
policy: ScalingPolicy,
snapshot: ScalingSnapshot,
now: number
): ScalingDecision {
if (policy.mapSlug === '*') {
const maps = zombieMaps(policy.game);
const decisions = maps.map(map => {
const matches = (name: string) => (snapshot.members.find(m => m.name === name)?.mapSlug ?? parseServerName(name)?.mapSlug) === map.slug;
return planScaling({ ...policy, mapSlug: map.slug }, {
...snapshot, members: snapshot.members.filter(m => matches(m.name)),
reservedNames: snapshot.reservedNames.filter(matches)
}, now);
const decisions = maps.map((map) => {
const matches = (name: string) =>
(snapshot.members.find((m) => m.name === name)?.mapSlug ??
parseServerName(name)?.mapSlug) === map.slug;
return planScaling(
{ ...policy, mapSlug: map.slug },
{
...snapshot,
members: snapshot.members.filter((m) => matches(m.name)),
reservedNames: snapshot.reservedNames.filter(matches)
},
now
);
});
// Serialize game-wide work and prefer replenishing any map before trimming another.
const pending = snapshot.members.find(m => m.state === 'creating' || m.state === 'removing');
const selected = decisions.find(d => d.action === 'create') ?? decisions.find(d => d.action === 'destroy')
?? decisions.find(d => d.deficit > 0 || d.unknown > 0) ?? decisions[0];
const pending = snapshot.members.find(
(m) => m.state === 'creating' || m.state === 'removing'
);
const selected =
decisions.find((d) => d.action === 'create') ??
decisions.find((d) => d.action === 'destroy') ??
decisions.find((d) => d.deficit > 0 || d.unknown > 0) ??
decisions[0];
const map = maps[decisions.indexOf(selected)];
return { ...selected, action: pending ? 'wait' : selected.action, server: pending ? null : selected.server,
reason: pending ? pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.` : `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0), deficit: decisions.reduce((n, d) => n + d.deficit, 0), occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0), pending: decisions.reduce((n, d) => n + d.pending, 0), unknown: decisions.reduce((n, d) => n + d.unknown, 0) };
return {
...selected,
action: pending ? 'wait' : selected.action,
server: pending ? null : selected.server,
reason: pending
? (pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.`)
: `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0),
deficit: decisions.reduce((n, d) => n + d.deficit, 0),
occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0),
pending: decisions.reduce((n, d) => n + d.pending, 0),
unknown: decisions.reduce((n, d) => n + d.unknown, 0)
};
}
const { members } = snapshot;
const occupied = members.filter(m => m.state === 'occupied').length, empty = members.filter(m => m.state === 'empty').length;
const pending = members.filter(m => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter(m => m.state === 'unknown' || m.state === 'stopped').length;
const occupied = members.filter((m) => m.state === 'occupied').length,
empty = members.filter((m) => m.state === 'empty').length;
const pending = members.filter((m) => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter((m) => m.state === 'unknown' || m.state === 'stopped').length;
const desired = Math.min(policy.maximum, Math.max(policy.minimum, occupied + policy.spares));
const result = (action: ScalingDecision['action'], reason: string, server: string | null = null): ScalingDecision =>
({ action, reason, server, desired, deficit: Math.max(0, policy.spares - empty), occupied, empty, pending, unknown });
const result = (
action: ScalingDecision['action'],
reason: string,
server: string | null = null
): ScalingDecision => ({
action,
reason,
server,
desired,
deficit: Math.max(0, policy.spares - empty),
occupied,
empty,
pending,
unknown
});
if (policy.mode === 'off') return result('wait', policy.pausedReason ?? 'Policy is off.');
if (policy.pausedReason) return result('wait', policy.pausedReason);
if (snapshot.blockers.length) return result('wait', snapshot.blockers[0]);
const blocked = members.find(m => !['occupied', 'empty'].includes(m.state));
if (blocked) return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
if (members.some(m => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS)) return result('wait', 'Waiting for fresh occupancy evidence.');
const blocked = members.find((m) => !['occupied', 'empty'].includes(m.state));
if (blocked)
return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
if (
members.some(
(m) => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS
)
)
return result('wait', 'Waiting for fresh occupancy evidence.');
if (occupied + empty < desired) {
if (snapshot.reservedNames.length >= policy.maximum) return result('wait', 'Maximum lobby reservations reached.');
if (!snapshot.keyPool.available) return result('wait', snapshot.keyPool.total
? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.`
: `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`);
const available = Array.from({ length: 20 }, (_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`)
.find(name => !snapshot.reservedNames.includes(name));
return available ? result('create', 'Replenish empty lobby capacity.', available) : result('wait', 'No unused lobby identity is available.');
if (snapshot.reservedNames.length >= policy.maximum)
return result('wait', 'Maximum lobby reservations reached.');
if (!snapshot.keyPool.available)
return result(
'wait',
snapshot.keyPool.total
? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.`
: `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`
);
const available = Array.from(
{ length: 20 },
(_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`
).find((name) => !snapshot.reservedNames.includes(name));
return available
? result('create', 'Replenish empty lobby capacity.', available)
: result('wait', 'No unused lobby identity is available.');
}
if (occupied + empty > desired && empty > policy.spares && occupied + empty > policy.minimum) {
// Protect the lowest-numbered spares before checking removal timers.
// Lobby 1 must survive even when a higher lobby has only just emptied.
const orderedEmpty = members.filter(m => m.state === 'empty').sort((a, b) =>
(parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) - (parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER)
|| a.name.localeCompare(b.name, 'en', { numeric: true }));
const orderedEmpty = members
.filter((m) => m.state === 'empty')
.sort(
(a, b) =>
(parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) -
(parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER) ||
a.name.localeCompare(b.name, 'en', { numeric: true })
);
const requiredEmpty = Math.max(policy.spares, policy.minimum - occupied);
const candidates = orderedEmpty.slice(requiredEmpty).reverse()
.filter(m => m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000);
if (candidates.length) return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
const candidates = orderedEmpty
.slice(requiredEmpty)
.reverse()
.filter(
(m) =>
m.emptySince !== null &&
m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000
);
if (candidates.length)
return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
return result('wait', 'Waiting for a continuous empty interval.');
}
return result('wait', empty < policy.spares ? 'Maximum reached; the empty spare target cannot be met.' : 'Empty spare target is satisfied.');
return result(
'wait',
empty < policy.spares
? 'Maximum reached; the empty spare target cannot be met.'
: 'Empty spare target is satisfied.'
);
}
src/lib/server/autoscaling/schema.ts
+13 -5
View File
@@ -36,20 +36,28 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
`);
handle.transaction(() => {
const columns = handle.pragma('table_info(autoscaling_policies)') as { name: string }[];
if (columns.some(column => column.name === 'next_action_at')) {
if (columns.some((column) => column.name === 'next_action_at')) {
handle.exec('ALTER TABLE autoscaling_policies DROP COLUMN next_action_at');
}
handle.exec(`UPDATE autoscaling_policies SET settings=json_remove(settings,'$.cooldownMinutes')
WHERE json_type(settings,'$.cooldownMinutes') IS NOT NULL`);
// Removed modes must not silently gain permission to destroy lobbies.
// Keep ownership while delivered work settles through normal reconciliation.
handle.prepare(`UPDATE autoscaling_authorizations SET revoked_at=?
handle
.prepare(
`UPDATE autoscaling_authorizations SET revoked_at=?
WHERE claimed_at IS NULL AND revoked_at IS NULL AND policy_id IN
(SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`).run(Date.now());
handle.prepare(`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'),
(SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`
)
.run(Date.now());
handle
.prepare(
`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'),
revision=revision+1, paused_reason='This autoscaling mode was removed. Review and turn autoscaling On to resume.',
owns_map=CASE WHEN EXISTS (SELECT 1 FROM autoscaling_decisions WHERE policy_id=autoscaling_policies.id AND resolved=0)
THEN owns_map ELSE 0 END, updated_at=?
WHERE mode IN ('dry-run','create-only')`).run(new Date().toISOString());
WHERE mode IN ('dry-run','create-only')`
)
.run(new Date().toISOString());
})();
}
src/lib/server/autoscaling/snapshot.ts
+189 -45
View File
@@ -7,74 +7,218 @@ import { freshStatusTimestamp } from '../../server-freshness.js';
import type { ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import { emptyObservation, type EmptyObservation } from './planner.js';
export function mapIdentity(server: { name: string; game?: string | null; map_slug?: string | null; map?: string | null }): { game: string; mapSlug: string } | null {
const parsed = parseServerName(server.name), game = server.game ?? parsed?.game;
export function mapIdentity(server: {
name: string;
game?: string | null;
map_slug?: string | null;
map?: string | null;
}): { game: string; mapSlug: string } | null {
const parsed = parseServerName(server.name),
game = server.game ?? parsed?.game;
if (!game) return null;
if (server.map_slug && zombieMaps(game).some(m => m.slug === server.map_slug)) return { game, mapSlug: server.map_slug };
if (server.map_slug && zombieMaps(game).some((m) => m.slug === server.map_slug))
return { game, mapSlug: server.map_slug };
if (server.map_slug) return null;
if (parsed?.game === game && parsed.mapCanonical && (!server.map || parsed.mapCanonical === server.map)) return { game, mapSlug: parsed.mapSlug };
const matches = zombieMaps(game).filter(m => m.canonical === server.map);
if (
parsed?.game === game &&
parsed.mapCanonical &&
(!server.map || parsed.mapCanonical === server.map)
)
return { game, mapSlug: parsed.mapSlug };
const matches = zombieMaps(game).filter((m) => m.canonical === server.map);
return matches.length === 1 ? { game, mapSlug: matches[0].slug } : null;
}
export function protectedWork(server: string, exceptOperation = ''): string | null {
const handle = db();
if (handle.prepare(`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).get(server)) return 'Active beta';
if (handle.prepare(`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).get(server)) return 'Campaign work';
if (handle.prepare(`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`).get(server)) return 'Maintenance work';
if (handle.prepare(`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`).get(server, exceptOperation)) return 'Unresolved recovery';
if (
handle
.prepare(
`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`
)
.get(server)
)
return 'Active beta';
if (
handle
.prepare(
`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`
)
.get(server)
)
return 'Campaign work';
if (
handle
.prepare(
`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`
)
.get(server)
)
return 'Maintenance work';
if (
handle
.prepare(
`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`
)
.get(server, exceptOperation)
)
return 'Unresolved recovery';
return null;
}
export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptOperation = ''): ScalingSnapshot {
const handle = db(), observe = observationReader();
export function scalingSnapshot(
policy: ScalingPolicy,
now = Date.now(),
exceptOperation = ''
): ScalingSnapshot {
const handle = db(),
observe = observationReader();
const rows = listServers();
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && (policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug); };
const matches = (value: Parameters<typeof mapIdentity>[0]) => {
const identity = mapIdentity(value);
return (
identity?.game === policy.game &&
(policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug)
);
};
const blockers: string[] = [];
for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string; stage: string }[];
const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
for (const row of rows)
if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game)
blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle
.prepare(
`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`
)
.all(exceptOperation) as {
id: string;
server: string;
action: string;
state: string;
stage: string;
}[];
const reservations = handle
.prepare(
`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`
)
.all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => {
const observation = observe(row), operation = operations.find(o => o.server === row.name);
const observation = observe(row),
operation = operations.find((o) => o.server === row.name);
const protection = protectedWork(row.name, exceptOperation);
const host = row.host_id ? handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as { mode: string } | undefined : undefined;
const host = row.host_id
? (handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as
{ mode: string } | undefined)
: undefined;
const identity = `${row.native_instance_id ?? row.created_at}:${row.instance_generation ?? 0}:${observation.telemetry_after ?? ''}`;
let state: ScalingMember['state'] = 'unknown', blocker: string | null = null;
let state: ScalingMember['state'] = 'unknown',
blocker: string | null = null;
let players = observation.health === 'online' ? observation.last_players : null;
let observedAt = players !== null && observation.last_seen_at ? Date.parse(observation.last_seen_at) : null;
let observedAt =
players !== null && observation.last_seen_at
? Date.parse(observation.last_seen_at)
: null;
let source = observation.status_source;
// Display evidence prefers telemetry; scaling additionally honors a current positive RCON veto.
if ((row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now)
&& (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) {
players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon';
if (
(row.last_players ?? 0) > 0 &&
freshStatusTimestamp(row.last_seen_at, now) &&
(!observation.telemetry_after ||
Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))
) {
players = row.last_players;
observedAt = Date.parse(row.last_seen_at!);
source = 'rcon';
}
if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; }
else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id}: ${operation.stage}`; }
else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
return { name: row.name, mapSlug: mapIdentity(row)!.mapSlug, identity, state, players, observedAt, source: source ?? null, emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
if (protection || !row.native_instance_id || host?.mode !== 'active') {
state = 'protected';
blocker = protection ?? 'Native instance on an active host required';
} else if (operation) {
state =
operation.action === 'create'
? 'creating'
: operation.action === 'destroy'
? 'removing'
: 'protected';
blocker = `Provisioning ${operation.id}: ${operation.stage}`;
} else if (
row.desired_state === 'stopped' ||
['stopped', 'exited', 'created', 'dead'].includes(row.container_state ?? '')
)
state = 'stopped';
else if (
observation.health === 'online' &&
players !== null &&
observedAt !== null &&
observedAt <= now &&
freshStatusTimestamp(new Date(observedAt).toISOString(), now)
)
state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since]
.map((at) => Date.parse(at ?? ''))
.filter((at) => Number.isFinite(at) && at <= now);
return {
name: row.name,
mapSlug: mapIdentity(row)!.mapSlug,
identity,
state,
players,
observedAt,
source: source ?? null,
emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null,
blocker,
operationId: operation?.id ?? null
};
});
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
const operation = operations.find(o => o.server === row.name);
members.push({ name: row.name, mapSlug: row.map_slug, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
}
const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
for (const row of reservations.filter(matches))
if (!members.some((m) => m.name === row.name)) {
const operation = operations.find((o) => o.server === row.name);
members.push({
name: row.name,
mapSlug: row.map_slug,
identity: 'reserved',
state: 'creating',
players: null,
observedAt: null,
source: null,
emptySince: null,
blocker: operation?.stage ?? 'Creation is pending.',
operationId: operation?.id ?? null
});
}
const retained = handle
.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?')
.all(policy.id) as { server: string; observation: string }[];
for (const member of members) {
const saved = retained.find(r => r.server === member.name);
const previous = saved ? JSON.parse(saved.observation) as EmptyObservation : undefined;
const saved = retained.find((r) => r.server === member.name);
const previous = saved ? (JSON.parse(saved.observation) as EmptyObservation) : undefined;
member.emptySince = emptyObservation(member, previous, now).emptySince;
}
const reservedNames = new Set([...rows.filter(matches).map(r => r.name), ...reservations.filter(matches).map(r => r.name)]);
for (const row of handle.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL').all() as { name: string }[]) if (matches(row)) {
reservedNames.add(row.name);
if (!members.some(m => m.name === row.name)) blockers.push(`Unresolved native instance ${row.name}.`);
}
for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as { name: string }[]) if (matches(row)) reservedNames.add(row.name);
const ownServer = exceptOperation ? (handle.prepare('SELECT server FROM server_deployments WHERE id=?').get(exceptOperation) as { server: string } | undefined)?.server : undefined;
const reservedNames = new Set([
...rows.filter(matches).map((r) => r.name),
...reservations.filter(matches).map((r) => r.name)
]);
for (const row of handle
.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL')
.all() as { name: string }[])
if (matches(row)) {
reservedNames.add(row.name);
if (!members.some((m) => m.name === row.name))
blockers.push(`Unresolved native instance ${row.name}.`);
}
for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as {
name: string;
}[])
if (matches(row)) reservedNames.add(row.name);
const ownServer = exceptOperation
? (
handle
.prepare('SELECT server FROM server_deployments WHERE id=?')
.get(exceptOperation) as { server: string } | undefined
)?.server
: undefined;
const { total, available } = serverKeyPool(policy.game, ownServer);
return { members, reservedNames: [...reservedNames], keyPool: { total, available }, blockers };
}
src/lib/server/backends/files.ts
+107 -29
View File
@@ -16,28 +16,65 @@ function remote(path: string) {
}
return null;
}
function missing(path: string): never { throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' }); }
function missing(path: string): never {
throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' });
}
export interface FileFingerprint { size: number; sha256: string }
export interface FileFingerprint {
size: number;
sha256: string;
}
/** One scoped agent scan; null preserves compatibility with older agents and local files. */
export async function nativeFileManifest(root: string, directories: string[], ignoreNames: string[], managedPaths: string[]): Promise<Map<string, FileFingerprint> | null> {
export async function nativeFileManifest(
root: string,
directories: string[],
ignoreNames: string[],
managedPaths: string[]
): Promise<Map<string, FileFingerprint> | null> {
const target = remote(root);
if (!target || target.path !== '.' || !getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest')) return null;
const roots = [...new Set(directories.map(directory => relative(root, directory).split(sep).join('/')))];
if (
!target ||
target.path !== '.' ||
!getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest')
)
return null;
const roots = [
...new Set(directories.map((directory) => relative(root, directory).split(sep).join('/')))
];
// Preserve legacy configurations that scan the entire instance root.
if (roots.some(directory => !directory)) return null;
const paths = [...new Set(managedPaths.map(path => relative(root, path).split(sep).join('/')))];
const result = await nativeRequest<{ files: Array<FileFingerprint & { path: string }> }>(target.instance, 'files.manifest', {
roots, paths, ignoreNames: ignoreNames.filter(name => !paths.some(path => path.split('/').includes(name)))
});
if (!result || !Array.isArray(result.files) || result.files.length > 10_000) throw new Error('Agent returned an invalid file manifest.');
if (roots.some((directory) => !directory)) return null;
const paths = [
...new Set(managedPaths.map((path) => relative(root, path).split(sep).join('/')))
];
const result = await nativeRequest<{ files: Array<FileFingerprint & { path: string }> }>(
target.instance,
'files.manifest',
{
roots,
paths,
ignoreNames: ignoreNames.filter(
(name) => !paths.some((path) => path.split('/').includes(name))
)
}
);
if (!result || !Array.isArray(result.files) || result.files.length > 10_000)
throw new Error('Agent returned an invalid file manifest.');
const manifest = new Map<string, FileFingerprint>();
for (const entry of result.files) {
if (!entry || typeof entry.path !== 'string' || !entry.path || entry.path.includes('\\') || entry.path.includes('\0')
|| posix.normalize(entry.path) !== entry.path || entry.path.split('/').some(part => part === '..' || part === '.')
|| !roots.some(directory => entry.path.startsWith(`${directory}/`))
|| !Number.isSafeInteger(entry.size) || entry.size < 0 || !/^[a-f0-9]{64}$/.test(entry.sha256)) {
if (
!entry ||
typeof entry.path !== 'string' ||
!entry.path ||
entry.path.includes('\\') ||
entry.path.includes('\0') ||
posix.normalize(entry.path) !== entry.path ||
entry.path.split('/').some((part) => part === '..' || part === '.') ||
!roots.some((directory) => entry.path.startsWith(`${directory}/`)) ||
!Number.isSafeInteger(entry.size) ||
entry.size < 0 ||
!/^[a-f0-9]{64}$/.test(entry.sha256)
) {
throw new Error('Agent returned an invalid file manifest entry.');
}
const path = join(root, entry.path);
@@ -50,44 +87,85 @@ export async function nativeFileManifest(root: string, directories: string[], ig
export async function readFile(path: string): Promise<Buffer> {
const target = remote(path);
if (!target) return fs.readFile(path);
const result = await nativeRequest<{ contentSha256?: string; contentBase64?: string | null } | null>(target.instance, 'files.read', { path: target.path });
const result = await nativeRequest<{
contentSha256?: string;
contentBase64?: string | null;
} | null>(target.instance, 'files.read', { path: target.path });
// Stored receipts reference shared bytes; read them directly without a base64 round trip.
if (typeof result?.contentSha256 === 'string') return getFileBlob(result.contentSha256);
if (result?.contentBase64 === null) return missing(path);
if (typeof result?.contentBase64 === 'string') return Buffer.from(result.contentBase64, 'base64');
if (typeof result?.contentBase64 === 'string')
return Buffer.from(result.contentBase64, 'base64');
throw new Error(`Agent returned an invalid response for files.read: ${target.path}`);
}
export async function exists(path: string): Promise<boolean> {
try { await stat(path); return true; }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; }
try {
await stat(path);
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
throw error;
}
}
export async function stat(path: string): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> {
export async function stat(
path: string
): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> {
const target = remote(path);
if (!target) return fs.stat(path);
const result = await nativeRequest<{ exists?: boolean; size: number; mtime: string | number; isDirectory: boolean; isFile: boolean } | null>(target.instance, 'files.stat', { path: target.path });
const result = await nativeRequest<{
exists?: boolean;
size: number;
mtime: string | number;
isDirectory: boolean;
isFile: boolean;
} | null>(target.instance, 'files.stat', { path: target.path });
if (result === null || result.exists === false) return missing(path);
return { size: result.size, mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime), isDirectory: () => result.isDirectory, isFile: () => result.isFile };
return {
size: result.size,
mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime),
isDirectory: () => result.isDirectory,
isFile: () => result.isFile
};
}
export async function readdir(path: string, options: { withFileTypes: true }): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> {
export async function readdir(
path: string,
options: { withFileTypes: true }
): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> {
const target = remote(path);
if (!target) return fs.readdir(path, options);
const result = await nativeRequest<{ entries: { name: string; isDirectory: boolean; isFile?: boolean }[] }>(target.instance, 'files.list', { path: target.path });
return result.entries.map(entry => ({ name: entry.name, isDirectory: () => entry.isDirectory, isFile: () => entry.isFile ?? !entry.isDirectory }));
const result = await nativeRequest<{
entries: { name: string; isDirectory: boolean; isFile?: boolean }[];
}>(target.instance, 'files.list', { path: target.path });
return result.entries.map((entry) => ({
name: entry.name,
isDirectory: () => entry.isDirectory,
isFile: () => entry.isFile ?? !entry.isDirectory
}));
}
export async function nativeTail(path: string, lines: number) {
const target = remote(path);
if (!target) return undefined;
return nativeRequest<import('../fleet/logs.js').TailResult | null>(target.instance, 'logs.read', { path: target.path, lines });
return nativeRequest<import('../fleet/logs.js').TailResult | null>(
target.instance,
'logs.read',
{ path: target.path, lines }
);
}
export async function nativeLogChunk(path: string, cursor?: import('../fleet/logs.js').LogCursor) {
const target = remote(path);
if (!target) return undefined;
const incremental = getHost(target.instance.host_id)?.inventory.capabilities?.includes('logs.read-offset');
return nativeRequest<import('../fleet/logs.js').LogChunk | import('../fleet/logs.js').TailResult | null>(target.instance, 'logs.read',
{ path: target.path, ...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 }) });
const incremental = getHost(target.instance.host_id)?.inventory.capabilities?.includes(
'logs.read-offset'
);
return nativeRequest<
import('../fleet/logs.js').LogChunk | import('../fleet/logs.js').TailResult | null
>(target.instance, 'logs.read', {
path: target.path,
...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 })
});
}
src/lib/server/backends/native.ts
+105 -28
View File
@@ -6,13 +6,26 @@ import { trackActivity } from '../pipeline/activity.js';
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
export interface NativeInstance {
id: string; server: string; host_id: string; generation: number; library_id: string;
image: string; bind_address: string; port: number; public_address: string | null;
public_port: number | null; created_at: string; retired_at: string | null;
id: string;
server: string;
host_id: string;
generation: number;
library_id: string;
image: string;
bind_address: string;
port: number;
public_address: string | null;
public_port: number | null;
created_at: string;
retired_at: string | null;
}
export function nativeInstance(server: string): NativeInstance | null {
return db().prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL').get(server) as NativeInstance | undefined ?? null;
return (
(db()
.prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL')
.get(server) as NativeInstance | undefined) ?? null
);
}
/** Credentials come from the launch, never from the possibly modified config or
@@ -21,72 +34,136 @@ export function nativeInstance(server: string): NativeInstance | null {
export function runtimePasswordForServer(server: string): string | null {
const instance = nativeInstance(server);
if (!instance) return null;
const row = db().prepare(`SELECT
const row = db()
.prepare(
`SELECT
json_extract(COALESCE(p.payload,o.payload),'$.rconPassword') AS password,
json_extract(COALESCE(p.payload,o.payload),'$.env.PLUTAINER_RCON_PASSWORD') AS env_password
FROM ec_host_operations o JOIN native_instances i ON ${runtimeLaunchSql}
LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id
WHERE i.id=? AND o.state='completed' AND o.history_expired_at IS NULL
ORDER BY o.rowid DESC LIMIT 1`).get(instance.id) as { password: unknown; env_password: unknown } | undefined;
if (!row) throw new Error(`Cannot verify runtime config for ${server}: the current instance's launch credentials are unavailable.`);
ORDER BY o.rowid DESC LIMIT 1`
)
.get(instance.id) as { password: unknown; env_password: unknown } | undefined;
if (!row)
throw new Error(
`Cannot verify runtime config for ${server}: the current instance's launch credentials are unavailable.`
);
const password = row.password || row.env_password;
return typeof password === 'string' && password ? password : null;
}
export function nativeDeployment(id: string): boolean {
return !!db().prepare(`SELECT 1 FROM native_provisioning WHERE deployment_id = ?
return !!db()
.prepare(
`SELECT 1 FROM native_provisioning WHERE deployment_id = ?
UNION ALL SELECT 1 FROM native_creation_intents WHERE deployment_id = ?
UNION ALL SELECT 1 FROM native_creation_cancellations c
JOIN native_creation_intents i ON i.deployment_id=c.creation_id WHERE c.cleanup_id = ? LIMIT 1`).get(id, id, id);
JOIN native_creation_intents i ON i.deployment_id=c.creation_id WHERE c.cleanup_id = ? LIMIT 1`
)
.get(id, id, id);
}
export function unplacedNativeCreation(server: string): string | null {
const row = db().prepare(`SELECT d.id FROM server_deployments d
const row = db()
.prepare(
`SELECT d.id FROM server_deployments d
JOIN native_creation_intents i ON i.deployment_id=d.id
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
WHERE d.server=? AND d.server_deleted_at IS NULL AND d.action='create' AND d.state='queued'
AND n.deployment_id IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(server) as { id: string } | undefined;
AND n.deployment_id IS NULL ORDER BY d.rowid DESC LIMIT 1`
)
.get(server) as { id: string } | undefined;
return row?.id ?? null;
}
const observations = new Map<string, Promise<unknown>>();
const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'files.manifest', 'logs.read']);
const readKinds = new Set([
'instance.inspect',
'files.read',
'files.stat',
'files.list',
'files.manifest',
'logs.read'
]);
/** Share overlapping observations only, never completed results or mutations. */
export async function nativeRequest<T = Record<string, unknown>>(instance: NativeInstance, kind: string, payload: Record<string, unknown> = {}, key?: string): Promise<T> {
export async function nativeRequest<T = Record<string, unknown>>(
instance: NativeInstance,
kind: string,
payload: Record<string, unknown> = {},
key?: string
): Promise<T> {
const host = getHost(instance.host_id);
if (!host || host.connectionState !== 'online') throw new Error('Host agent is offline; instance state is unknown.');
const readOnly = readKinds.has(kind) || (kind === 'rcon.execute' && ['status', 'getstatus', 'version'].includes(String(payload.command)));
if (!host || host.connectionState !== 'online')
throw new Error('Host agent is offline; instance state is unknown.');
const readOnly =
readKinds.has(kind) ||
(kind === 'rcon.execute' &&
['status', 'getstatus', 'version'].includes(String(payload.command)));
if (key || !readOnly) return executeRequest<T>(instance, kind, payload, key);
// A probe after a lifecycle submission must not join an old-process read.
const identity = JSON.stringify([instance.host_id, instance.id, instance.generation, instance.server,
latestGameSubmission(instance.server), kind, payload]);
const identity = JSON.stringify([
instance.host_id,
instance.id,
instance.generation,
instance.server,
latestGameSubmission(instance.server),
kind,
payload
]);
const pending = observations.get(identity);
if (pending) return pending as Promise<T>;
const request = executeRequest<T>(instance, kind, payload).finally(() => { observations.delete(identity); });
const request = executeRequest<T>(instance, kind, payload).finally(() => {
observations.delete(identity);
});
observations.set(identity, request);
return request;
}
/** Only the agent receives host paths, Docker access, or game credentials. */
async function executeRequest<T>(instance: NativeInstance, kind: string, payload: Record<string, unknown>, key?: string): Promise<T> {
async function executeRequest<T>(
instance: NativeInstance,
kind: string,
payload: Record<string, unknown>,
key?: string
): Promise<T> {
const operation = queueHostOperation(instance.host_id, kind, payload, {
instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID()
instanceId: instance.id,
generation: instance.generation,
idempotencyKey: key ?? randomUUID()
});
const result = await trackActivity(`agent:${operation.id}`, `Waiting for ${kind} on ${instance.server}`,
() => waitHostOperationStatus(operation.id, 25_000));
if (result.state !== 'completed') throw new Error(result.error || `Agent operation ${result.id} is ${result.state}; its original ID is retained.`);
const result = await trackActivity(
`agent:${operation.id}`,
`Waiting for ${kind} on ${instance.server}`,
() => waitHostOperationStatus(operation.id, 25_000)
);
if (result.state !== 'completed')
throw new Error(
result.error ||
`Agent operation ${result.id} is ${result.state}; its original ID is retained.`
);
// Do not apply replies from a retired generation to a replacement server.
const current = nativeInstance(instance.server);
if (current?.id !== instance.id || current.generation !== instance.generation) throw new Error('Instance was replaced during the operation.');
if (current?.id !== instance.id || current.generation !== instance.generation)
throw new Error('Instance was replaced during the operation.');
return result.result as T;
}
export async function nativeHealth(server: string): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> {
export async function nativeHealth(
server: string
): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> {
const instance = nativeInstance(server);
if (!instance) return { ok: false, reason: 'Native instance is not registered' };
try {
const result = await nativeRequest<{ state: string; healthy: boolean }>(instance, 'instance.inspect');
return result.healthy ? { ok: true, state: result.state } : { ok: false, state: result.state, reason: `Container ${result.state}` };
} catch (error) { return { ok: false, state: 'unknown', reason: (error as Error).message }; }
const result = await nativeRequest<{ state: string; healthy: boolean }>(
instance,
'instance.inspect'
);
return result.healthy
? { ok: true, state: result.state }
: { ok: false, state: result.state, reason: `Container ${result.state}` };
} catch (error) {
return { ok: false, state: 'unknown', reason: (error as Error).message };
}
}
src/lib/server/backends/schema.ts
+36 -12
View File
@@ -43,11 +43,21 @@ export function migrateNativeInstances(handle: Database.Database): void {
);
CREATE INDEX IF NOT EXISTS idx_ec_host_migrations_instance ON ec_host_migrations(instance_id);
`);
if (!(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some(column => column.name === 'runtime_password')) {
if (
!(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some(
(column) => column.name === 'runtime_password'
)
) {
handle.exec('ALTER TABLE ec_host_migrations ADD COLUMN runtime_password TEXT');
}
if (!(handle.pragma('table_info(native_provisioning)') as { name: string }[]).some(column => column.name === 'recovery_step')) {
handle.exec("ALTER TABLE native_provisioning ADD COLUMN recovery_step TEXT CHECK (recovery_step IN ('restart','rebuild'))");
if (
!(handle.pragma('table_info(native_provisioning)') as { name: string }[]).some(
(column) => column.name === 'recovery_step'
)
) {
handle.exec(
"ALTER TABLE native_provisioning ADD COLUMN recovery_step TEXT CHECK (recovery_step IN ('restart','rebuild'))"
);
}
if (!handle.prepare("SELECT 1 FROM settings WHERE key='native_file_plans_v1'").get()) {
handle.transaction(() => {
@@ -56,29 +66,43 @@ export function migrateNativeInstances(handle: Database.Database): void {
INSERT INTO settings(key,value) VALUES('native_file_plans_v1','1');`);
})();
}
}
/** Accepted native creations receive their host port only when provisioning places them. */
function migrateUnplacedCreationPort(handle: Database.Database): void {
const columns = handle.prepare('PRAGMA table_info(server_creation_jobs)').all() as { name: string; notnull: number }[];
if (!columns.find(column => column.name === 'port')?.notnull) return;
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='server_creation_jobs'").get() as { sql: string };
const columns = handle.prepare('PRAGMA table_info(server_creation_jobs)').all() as {
name: string;
notnull: number;
}[];
if (!columns.find((column) => column.name === 'port')?.notnull) return;
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='server_creation_jobs'")
.get() as { sql: string };
const replacement = sql.replace(/\bport\s+INTEGER\s+NOT\s+NULL\b/i, 'port INTEGER');
if (replacement === sql) throw new Error('Could not migrate unplaced creation ports.');
const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='server_creation_jobs' AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependent = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name='server_creation_jobs' AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all() as { sql: string }[];
const foreignKeys = handle.pragma('foreign_keys', { simple: true });
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement.replace('server_creation_jobs', 'server_creation_jobs_unplaced'));
const names = columns.map(column => `"${column.name.replaceAll('"', '""')}"`).join(', ');
handle.exec(
replacement.replace('server_creation_jobs', 'server_creation_jobs_unplaced')
);
const names = columns
.map((column) => `"${column.name.replaceAll('"', '""')}"`)
.join(', ');
handle.exec(`INSERT INTO server_creation_jobs_unplaced(rowid,${names}) SELECT rowid,${names} FROM server_creation_jobs;
DROP TABLE server_creation_jobs;
ALTER TABLE server_creation_jobs_unplaced RENAME TO server_creation_jobs;`);
for (const object of dependent) handle.exec(object.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Unplaced creation migration found broken references.');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Unplaced creation migration found broken references.');
})();
} finally { handle.pragma(`foreign_keys = ${foreignKeys ? 'ON' : 'OFF'}`); }
} finally {
handle.pragma(`foreign_keys = ${foreignKeys ? 'ON' : 'OFF'}`);
}
}
src/lib/server/config.ts
+17 -9
View File
@@ -17,13 +17,15 @@ function num(name: string, fallback: number): number {
export const config = {
serversRoot: req('SERVERS_ROOT', '/root/zombies/t5'),
t6ServersRoot: env.T6_SERVERS_ROOT ?? join(dirname(req('SERVERS_ROOT', '/root/zombies/t5')), 't6'),
t6ServersRoot:
env.T6_SERVERS_ROOT ?? join(dirname(req('SERVERS_ROOT', '/root/zombies/t5')), 't6'),
t6RelDestScript: env.T6_REL_DEST_SCRIPT ?? 'runtime/plutonium/storage/t6/scripts/zm',
t6ConfigTargetFile: env.T6_CONFIG_TARGET_FILE ?? 'dedicated_zm.cfg',
relDest: req('REL_DEST', 'runtime/plutonium/storage/t5/raw/scripts/sp'),
// Per-kind destination sub-paths under each server's directory. `relDest`
// above stays as the script fallback so existing deploys keep working.
relDestScript: env.REL_DEST_SCRIPT ?? env.REL_DEST ?? 'runtime/plutonium/storage/t5/raw/scripts/sp',
relDestScript:
env.REL_DEST_SCRIPT ?? env.REL_DEST ?? 'runtime/plutonium/storage/t5/raw/scripts/sp',
relDestConfig: env.REL_DEST_CONFIG ?? 'configs',
relDestPlugin: env.REL_DEST_PLUGIN ?? 'runtime/plutonium/plugins',
// Legacy T5 config filename. T6 has its own target above.
@@ -54,19 +56,25 @@ export type ArtifactKind = 'script' | 'config' | 'plugin' | 'baseline';
export function relDestForKind(kind: ArtifactKind, game = 't5'): string {
switch (kind) {
case 'script': return game === 't6' ? config.t6RelDestScript : config.relDestScript;
case 'config': return config.relDestConfig;
case 'baseline': return config.relDestConfig;
case 'plugin': return config.relDestPlugin;
case 'script':
return game === 't6' ? config.t6RelDestScript : config.relDestScript;
case 'config':
return config.relDestConfig;
case 'baseline':
return config.relDestConfig;
case 'plugin':
return config.relDestPlugin;
}
}
// Server IDs keep the game prefix even while creation is queued and no registry
// row exists yet. Legacy IDs retain the original T5 filesystem defaults.
export const gameOfServer = (server: string) => server.startsWith('t6_') ? 't6' : 't5';
export const rootOfServer = (server: string) => gameOfServer(server) === 't6' ? config.t6ServersRoot : config.serversRoot;
export const gameOfServer = (server: string) => (server.startsWith('t6_') ? 't6' : 't5');
export const rootOfServer = (server: string) =>
gameOfServer(server) === 't6' ? config.t6ServersRoot : config.serversRoot;
export const serverDir = (server: string) => `${rootOfServer(server)}/${server}`;
export const configFileForGame = (game: string | null) => game === 't6' ? config.t6ConfigTargetFile : config.configTargetFile;
export const configFileForGame = (game: string | null) =>
game === 't6' ? config.t6ConfigTargetFile : config.configTargetFile;
export const configFileOf = (server: string) => configFileForGame(gameOfServer(server));
export const destOfKind = (server: string, kind: ArtifactKind) =>
`${serverDir(server)}/${relDestForKind(kind, gameOfServer(server))}`;
src/lib/server/db/connection.ts
+4 -2
View File
@@ -26,8 +26,10 @@ export function migrationStep<T>(name: string, work: () => T): T {
console.info(`[database] migration starting: ${name}`);
try {
const result = work();
console.info(`[database] migration finished: ${name}`, { elapsedMs: Date.now() - started,
rssMiB: Math.round(process.memoryUsage().rss / 1048576) });
console.info(`[database] migration finished: ${name}`, {
elapsedMs: Date.now() - started,
rssMiB: Math.round(process.memoryUsage().rss / 1048576)
});
return result;
} catch (error) {
console.error(`[database] migration failed: ${name}`, error);
src/lib/server/db/file-blobs.ts
+101 -39
View File
@@ -10,67 +10,103 @@ export function ensureFileBlobSchema(handle: Database.Database): void {
CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256);`);
}
export function putFileBlob(content: Uint8Array | null, handle: Database.Database = db()): string | null {
export function putFileBlob(
content: Uint8Array | null,
handle: Database.Database = db()
): string | null {
if (content === null) return null;
const sha256 = createHash('sha256').update(content).digest('hex');
handle.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)').run(sha256, content.byteLength, content);
handle
.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)')
.run(sha256, content.byteLength, content);
return sha256;
}
export function getFileBlob(sha256: string, handle: Database.Database = db()): Buffer {
const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as { content: Buffer } | undefined;
const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as
{ content: Buffer } | undefined;
if (!row) throw new Error('File content is unavailable or has expired.');
return row.content;
}
/** SQL fragments are for fixed schema identifiers, never user input. */
export const blobContent = (column: string) => `(SELECT content FROM file_blobs WHERE sha256=${column})`;
export const blobContent = (column: string) =>
`(SELECT content FROM file_blobs WHERE sha256=${column})`;
export const blobSize = (column: string) => `(SELECT size FROM file_blobs WHERE sha256=${column})`;
export function referenceBlobs(ownerType: string, ownerId: string, hashes: Iterable<string>, handle: Database.Database = db()): void {
handle.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?').run(ownerType, ownerId);
const insert = handle.prepare('INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)');
export function referenceBlobs(
ownerType: string,
ownerId: string,
hashes: Iterable<string>,
handle: Database.Database = db()
): void {
handle
.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?')
.run(ownerType, ownerId);
const insert = handle.prepare(
'INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)'
);
for (const hash of hashes) insert.run(ownerType, ownerId, hash);
}
/** Immutable file bytes in JSON are replaced with hash references, including before images. */
export function packFileJson(value: unknown, handle: Database.Database = db(), hashes = new Set<string>()): { value: unknown; hashes: Set<string> } {
if (Array.isArray(value)) return { value: value.map(item => packFileJson(item, handle, hashes).value), hashes };
export function packFileJson(
value: unknown,
handle: Database.Database = db(),
hashes = new Set<string>()
): { value: unknown; hashes: Set<string> } {
if (Array.isArray(value))
return { value: value.map((item) => packFileJson(item, handle, hashes).value), hashes };
if (!value || typeof value !== 'object') return { value, hashes };
const result: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) {
if ((key === 'contentBase64' || key === 'content' && 'path' in value && 'sha256' in value) && typeof item === 'string') {
if (
(key === 'contentBase64' ||
(key === 'content' && 'path' in value && 'sha256' in value)) &&
typeof item === 'string'
) {
const hash = putFileBlob(Buffer.from(item, 'base64'), handle)!;
result[key === 'content' ? 'contentHash' : 'contentSha256'] = hash;
hashes.add(hash);
} else {
result[key] = packFileJson(item, handle, hashes).value;
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') hashes.add(item);
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string')
hashes.add(item);
}
}
return { value: result, hashes };
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Unpacked JSON retains the caller-specific stored object shape.
export function unpackFileJson(value: unknown, handle: Database.Database = db()): any {
if (Array.isArray(value)) return value.map(item => unpackFileJson(item, handle));
if (Array.isArray(value)) return value.map((item) => unpackFileJson(item, handle));
if (!value || typeof value !== 'object') return value;
const result: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) {
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') {
result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(item, handle).toString('base64');
result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(
item,
handle
).toString('base64');
} else result[key] = unpackFileJson(item, handle);
}
return result;
}
export const blobColumns = [
['script_versions', 'content'], ['staged_file_changes', 'content'],
['deployment_files', 'content_before'], ['deployment_files', 'content_after'],
['beta_file_snapshots', 'prior_content'], ['beta_revision_files', 'content']
['script_versions', 'content'],
['staged_file_changes', 'content'],
['deployment_files', 'content_before'],
['deployment_files', 'content_after'],
['beta_file_snapshots', 'prior_content'],
['beta_revision_files', 'content']
] as const;
/** Finish each SELECT before writing; better-sqlite3 iterators hold an active cursor. */
export function* migrationRows<T>(handle: Database.Database, sql: string): Generator<T & { rid: number }> {
export function* migrationRows<T>(
handle: Database.Database,
sql: string
): Generator<T & { rid: number }> {
const select = handle.prepare(sql);
let cursor = -Number.MAX_SAFE_INTEGER;
for (;;) {
@@ -84,31 +120,53 @@ export function* migrationRows<T>(handle: Database.Database, sql: string): Gener
/** Backfill one row at a time so large libraries do not exhaust startup memory. */
export function migrateFileBlobs(handle: Database.Database): void {
for (const [table, column] of blobColumns) {
if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue;
migrationStep(`shared files: ${table}.${column}`, () => handle.transaction(() => {
handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`);
const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) {
update.run(putFileBlob(row.content, handle), row.rid);
}
handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
})());
if (
!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(
(item) => item.name === column
)
)
continue;
migrationStep(`shared files: ${table}.${column}`, () =>
handle.transaction(() => {
handle.exec(
`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`
);
const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
for (const row of migrationRows<{ content: Buffer | null }>(
handle,
`SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`
)) {
update.run(putFileBlob(row.content, handle), row.rid);
}
handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
})()
);
}
for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`);
for (const [table, column] of blobColumns)
handle.exec(
`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`
);
if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) {
migrationStep('shared files: launch recipes', () => handle.transaction(() => {
for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle,
'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) {
const packed = packFileJson(JSON.parse(row.recipe), handle);
handle.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(packed.value), row.deployment_id);
referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
}
handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
})());
migrationStep('shared files: launch recipes', () =>
handle.transaction(() => {
for (const row of migrationRows<{ deployment_id: string; recipe: string }>(
handle,
'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1'
)) {
const packed = packFileJson(JSON.parse(row.recipe), handle);
handle
.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?')
.run(JSON.stringify(packed.value), row.deployment_id);
referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
}
handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
})()
);
}
handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN
DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`);
for (const action of ['INSERT', 'UPDATE OF recipe']) handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'}
for (const action of ['INSERT', 'UPDATE OF recipe'])
handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'}
AFTER ${action} ON server_launch_recipes BEGIN
DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=NEW.deployment_id;
INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256)
@@ -117,8 +175,12 @@ export function migrateFileBlobs(handle: Database.Database): void {
}
export function pruneFileBlobs(limit = 100): number {
return db().prepare(`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b
return db()
.prepare(
`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b
WHERE NOT EXISTS (SELECT 1 FROM file_blob_references r WHERE r.sha256=b.sha256)
${blobColumns.map(([table, column]) => `AND NOT EXISTS (SELECT 1 FROM ${table} WHERE ${column}_hash=b.sha256)`).join('\n')}
LIMIT ?)`).run(limit).changes;
LIMIT ?)`
)
.run(limit).changes;
}
src/lib/server/db/fleet-settings.ts
+15 -5
View File
@@ -1,26 +1,36 @@
import type Database from 'better-sqlite3';
import { defaultFleetSettings, validateFleetSettings, type FleetSettings } from '../../fleet-settings.js';
import {
defaultFleetSettings,
validateFleetSettings,
type FleetSettings
} from '../../fleet-settings.js';
export const FLEET_SETTINGS_KEY = 'fleet_settings';
/** Import legacy environment values once; persisted settings win on every later boot. */
export function initializeFleetSettings(handle: Database.Database): void {
const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
const saved = handle
.prepare('SELECT value FROM settings WHERE key = ?')
.get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
if (saved) {
const settings = JSON.parse(saved.value);
const upgraded = validateFleetSettings({ ...defaultFleetSettings, ...settings });
handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
handle
.prepare('UPDATE settings SET value = ? WHERE key = ?')
.run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
return;
}
const values = { ...defaultFleetSettings };
const legacy: Partial<Record<keyof FleetSettings, string>> = {
deployKeepDays: 'DEPLOY_KEEP_DAYS', restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY',
deployKeepDays: 'DEPLOY_KEEP_DAYS',
restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY',
createConcurrency: 'DEPLOYMENT_CREATE_CONCURRENCY'
};
for (const key of Object.keys(legacy) as (keyof FleetSettings)[]) {
const raw = process.env[legacy[key]!];
if (raw) values[key] = Number(raw);
}
handle.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)')
handle
.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)')
.run(FLEET_SETTINGS_KEY, JSON.stringify(validateFleetSettings(values)));
}
src/lib/server/db/index.ts
+274 -80
View File
@@ -28,9 +28,15 @@ let optimizeTimer: NodeJS.Timeout | undefined;
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
// backfill columns; this can.
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'container_checked_at', ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT' },
{
name: 'container_checked_at',
ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT'
},
{ name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
{ name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
{
name: 'desired_state',
ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))"
},
// Base columns that shipped in the very first schema stay here for parity
// with fresh installs — no ALTER is issued if they're already present.
{ name: 'game_port', ddl: 'ALTER TABLE servers ADD COLUMN game_port INTEGER' },
@@ -43,7 +49,10 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'last_roster', ddl: 'ALTER TABLE servers ADD COLUMN last_roster TEXT' },
{ name: 'last_seen_at', ddl: 'ALTER TABLE servers ADD COLUMN last_seen_at TEXT' },
{ name: 'last_error', ddl: 'ALTER TABLE servers ADD COLUMN last_error TEXT' },
{ name: 'rcon_failures', ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0' },
{
name: 'rcon_failures',
ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0'
},
{ name: 'last_reply_snippet', ddl: 'ALTER TABLE servers ADD COLUMN last_reply_snippet TEXT' },
// Script-scope columns — static per server, populated via server-edit UI.
{ name: 'game', ddl: 'ALTER TABLE servers ADD COLUMN game TEXT' },
@@ -78,7 +87,10 @@ function migrateServers(handle: Database.Database): void {
const retired = new Set(['komodo_type', 'komodo_name', 'komodo_server']);
handle.transaction(() => {
for (const column of rows) {
if (retired.has(column.name)) handle.exec(`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`);
if (retired.has(column.name))
handle.exec(
`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`
);
}
})();
}
@@ -98,8 +110,14 @@ const SCRIPT_VERSIONS_COLUMNS: Array<{ name: string; ddl: string }> = [
// the authoritative "promoted" signal), and any version already gated
// through a ready-promote beta will be re-marked next time that state
// machine fires.
{ name: 'state', ddl: "ALTER TABLE script_versions ADD COLUMN state TEXT NOT NULL DEFAULT 'untested'" },
{ name: 'kind', ddl: "ALTER TABLE script_versions ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" }
{
name: 'state',
ddl: "ALTER TABLE script_versions ADD COLUMN state TEXT NOT NULL DEFAULT 'untested'"
},
{
name: 'kind',
ddl: "ALTER TABLE script_versions ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'"
}
];
const BETA_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [
@@ -107,18 +125,40 @@ const BETA_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [
];
const DEPLOYMENT_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'kind', ddl: "ALTER TABLE deployment_scripts ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" }
{
name: 'kind',
ddl: "ALTER TABLE deployment_scripts ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'"
}
];
const DEPLOYMENTS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'script_id', ddl: 'ALTER TABLE deployments ADD COLUMN script_id INTEGER REFERENCES scripts(id)' },
{ name: 'script_version_id', ddl: 'ALTER TABLE deployments ADD COLUMN script_version_id INTEGER REFERENCES script_versions(id)' },
{ name: 'prior_version_id', ddl: 'ALTER TABLE deployments ADD COLUMN prior_version_id INTEGER REFERENCES script_versions(id)' },
{ name: 'beta_id', ddl: 'ALTER TABLE deployments ADD COLUMN beta_id INTEGER REFERENCES betas(id)' },
{ name: 'rolled_back_by', ddl: 'ALTER TABLE deployments ADD COLUMN rolled_back_by TEXT REFERENCES deployments(id)' }
{
name: 'script_id',
ddl: 'ALTER TABLE deployments ADD COLUMN script_id INTEGER REFERENCES scripts(id)'
},
{
name: 'script_version_id',
ddl: 'ALTER TABLE deployments ADD COLUMN script_version_id INTEGER REFERENCES script_versions(id)'
},
{
name: 'prior_version_id',
ddl: 'ALTER TABLE deployments ADD COLUMN prior_version_id INTEGER REFERENCES script_versions(id)'
},
{
name: 'beta_id',
ddl: 'ALTER TABLE deployments ADD COLUMN beta_id INTEGER REFERENCES betas(id)'
},
{
name: 'rolled_back_by',
ddl: 'ALTER TABLE deployments ADD COLUMN rolled_back_by TEXT REFERENCES deployments(id)'
}
];
function migrateTable(handle: Database.Database, table: string, cols: Array<{ name: string; ddl: string }>): void {
function migrateTable(
handle: Database.Database,
table: string,
cols: Array<{ name: string; ddl: string }>
): void {
const rows = handle.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
const existing = new Set(rows.map((r) => r.name));
for (const col of cols) {
@@ -169,14 +209,16 @@ function backfillMultiScriptJoins(handle: Database.Database): void {
// operator can promote/rollback in the old app before upgrading.
function dropLegacyBetaTables(handle: Database.Database): void {
const legacy = handle
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name IN ('betas','beta_scripts','beta_deploys')")
.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name IN ('betas','beta_scripts','beta_deploys')"
)
.all() as Array<{ name: string }>;
if (legacy.length === 0) return;
const legacyNames = legacy.map((r) => r.name);
// The old `betas` table had a `slug` column; the new one has `id` + `script_id`.
// If we see `slug`, we know it's the old shape.
const betasCols = handle.prepare("PRAGMA table_info(betas)").all() as Array<{ name: string }>;
const betasCols = handle.prepare('PRAGMA table_info(betas)').all() as Array<{ name: string }>;
const isLegacyShape = betasCols.some((c) => c.name === 'slug');
if (!isLegacyShape) return;
@@ -205,8 +247,7 @@ function dropLegacyBetaTables(handle: Database.Database): void {
// active betas aren't blocked.
function assertSnapshotSchemeSafe(handle: Database.Database): void {
const done = handle.prepare("SELECT value FROM settings WHERE key = 'snapshots_v1'").get() as
| { value: string }
| undefined;
{ value: string } | undefined;
if (done) return;
const active = handle
.prepare("SELECT id FROM betas WHERE state IN ('running','ready-promote') LIMIT 1")
@@ -227,19 +268,36 @@ function migrateGameTelemetry(handle: Database.Database): void {
handle.transaction(() => {
const convert = (raw: string): string => {
const plan = JSON.parse(raw) as BetaPlan;
plan.requirements = plan.requirements.map(r => {
plan.requirements = plan.requirements.map((r) => {
const { event: _event, ...rest } = r as BetaRequirement & { event?: string };
return ['feature', 'restart-budget', 'clean-logs'].includes(r.kind)
? { ...requirement('manual', r.id), label: `Review: ${r.label}`.slice(0, 200) } : rest;
? { ...requirement('manual', r.id), label: `Review: ${r.label}`.slice(0, 200) }
: rest;
});
return JSON.stringify(plan);
};
const plans = handle.prepare('SELECT beta_id, plan FROM beta_test_plans').all() as Array<{ beta_id: number; plan: string }>;
for (const row of plans) handle.prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?').run(convert(row.plan), row.beta_id);
const presets = handle.prepare('SELECT name, plan FROM beta_test_presets').all() as Array<{ name: string; plan: string }>;
for (const row of presets) handle.prepare('UPDATE beta_test_presets SET plan = ? WHERE name = ?').run(convert(row.plan), row.name);
handle.prepare(`UPDATE beta_test_runs SET metrics = ?, cursors = '{}' WHERE retired_at IS NULL
AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'))`).run(JSON.stringify(emptyMetrics()));
const plans = handle.prepare('SELECT beta_id, plan FROM beta_test_plans').all() as Array<{
beta_id: number;
plan: string;
}>;
for (const row of plans)
handle
.prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?')
.run(convert(row.plan), row.beta_id);
const presets = handle.prepare('SELECT name, plan FROM beta_test_presets').all() as Array<{
name: string;
plan: string;
}>;
for (const row of presets)
handle
.prepare('UPDATE beta_test_presets SET plan = ? WHERE name = ?')
.run(convert(row.plan), row.name);
handle
.prepare(
`UPDATE beta_test_runs SET metrics = ?, cursors = '{}' WHERE retired_at IS NULL
AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'))`
)
.run(JSON.stringify(emptyMetrics()));
handle.exec(`UPDATE beta_test_plans SET outcome = 'active', reason = 'Gameplay telemetry updated; collecting fresh script events'
WHERE outcome IN ('active','passed') AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'));
UPDATE script_versions SET state = 'untested' WHERE state = 'tested'
@@ -254,54 +312,96 @@ function migrateGameTelemetry(handle: Database.Database): void {
}
function migrateProvisioningActions(handle: Database.Database): void {
const table = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'").get() as { sql: string };
const table = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'"
)
.get() as { sql: string };
if (table.sql.includes("'rebuild'")) return;
// Rebuild the CHECK constraint without cascading into events, recipes, or
// collection members. Preserve rowids: they order same-server operations.
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(table.sql.replace('server_deployments', 'server_deployments_new')
.replace("'create', 'destroy'", "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"));
const columns = (handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[])
.map(column => `"${column.name.replaceAll('"', '""')}"`).join(', ');
handle.exec(
table.sql
.replace('server_deployments', 'server_deployments_new')
.replace(
"'create', 'destroy'",
"'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"
)
);
const columns = (
handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[]
)
.map((column) => `"${column.name.replaceAll('"', '""')}"`)
.join(', ');
handle.exec(`INSERT INTO server_deployments_new (rowid, ${columns})
SELECT rowid, ${columns} FROM server_deployments;
DROP TABLE server_deployments;
ALTER TABLE server_deployments_new RENAME TO server_deployments;`);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Provisioning migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Provisioning migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
/** Rebuild constrained tables while preserving links, indexes, triggers, and row order. */
function migrateWatchdogConstraints(handle: Database.Database): void {
for (const name of ['provisioning_sources', 'deployment_servers']) {
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
const replacement = name === 'provisioning_sources'
? (sql.includes("'autoscaling'") ? sql : sql.replace(/'campaign','beta','maintenance'(?:,'watchdog')?/, "'campaign','beta','maintenance','watchdog','autoscaling'"))
: sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
.get(name) as { sql: string };
const replacement =
name === 'provisioning_sources'
? sql.includes("'autoscaling'")
? sql
: sql.replace(
/'campaign','beta','maintenance'(?:,'watchdog')?/,
"'campaign','beta','maintenance','watchdog','autoscaling'"
)
: sql.replace(
/server\s+TEXT NOT NULL REFERENCES servers\(name\)/,
'server TEXT NOT NULL'
);
if (sql === replacement) continue;
const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependent = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all(name) as { sql: string }[];
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement.replace(name, name + '_watchdog_new'));
handle.exec(`INSERT INTO ${name}_watchdog_new (rowid, ${
(handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
}) SELECT rowid, ${
(handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
} FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};`);
handle.exec(
`INSERT INTO ${name}_watchdog_new (rowid, ${(
handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]
)
.map((col) => col.name)
.join(', ')}) SELECT rowid, ${(
handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]
)
.map((col) => col.name)
.join(
', '
)} FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};`
);
for (const object of dependent) handle.exec(object.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Watchdog migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Watchdog migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
}
function backfillDesiredState(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) return;
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get())
return;
handle.transaction(() => {
// A failed stop may have reached the host agent. Preserve that explicit intent.
handle.exec(`UPDATE servers SET desired_state = 'stopped' WHERE 'stop' = (
@@ -331,35 +431,80 @@ export function db(): Database.Database {
migrateProvisioningActions(handle);
migrateWatchdogConstraints(handle);
migrateTable(handle, 'deployment_files', [
{ name: 'snapshot_pending', ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0' }
{
name: 'snapshot_pending',
ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0'
}
]);
migrateTable(handle, 'restart_rollout', [
{
name: 'operation_id',
ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT'
}
]);
migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
migrateTable(handle, 'server_deployments', [
{ name: 'requested_by', ddl: 'ALTER TABLE server_deployments ADD COLUMN requested_by TEXT' },
{ name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
{ name: 'checked_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT' },
{
name: 'requested_by',
ddl: 'ALTER TABLE server_deployments ADD COLUMN requested_by TEXT'
},
{
name: 'submitted_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT'
},
{
name: 'checked_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT'
},
{ name: 'players', ddl: 'ALTER TABLE server_deployments ADD COLUMN players INTEGER' },
{ name: 'phase_key', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_key TEXT' },
{ name: 'phase_started_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT' },
{ name: 'observed_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT' },
{ name: 'observation_json', ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT' }
{
name: 'phase_started_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT'
},
{
name: 'observed_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT'
},
{
name: 'observation_json',
ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT'
}
]);
migrateTable(handle, 'server_deployment_events', [
{ name: 'kind', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN kind TEXT' },
{ name: 'phase_key', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT' },
{ name: 'details_json', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT' }
{
name: 'phase_key',
ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT'
},
{
name: 'details_json',
ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT'
}
]);
for (const table of ['deployment_servers', 'beta_servers', 'maintenance_jobs']) {
migrateTable(handle, table, [{ name: 'provisioning_id', ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` }]);
migrateTable(handle, table, [
{
name: 'provisioning_id',
ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT`
}
]);
}
migrateTable(handle, 'maintenance_jobs', [
{ name: 'kind', ddl: "ALTER TABLE maintenance_jobs ADD COLUMN kind TEXT NOT NULL DEFAULT 'legacy'" },
{ name: 'origin', ddl: "ALTER TABLE maintenance_jobs ADD COLUMN origin TEXT NOT NULL DEFAULT 'manual'" }
{
name: 'kind',
ddl: "ALTER TABLE maintenance_jobs ADD COLUMN kind TEXT NOT NULL DEFAULT 'legacy'"
},
{
name: 'origin',
ddl: "ALTER TABLE maintenance_jobs ADD COLUMN origin TEXT NOT NULL DEFAULT 'manual'"
}
]);
migrateServers(handle);
// Old snippets may contain player addresses, including hex-encoded bytes.
// Discard them once; newly retained diagnostics are sanitized failure replies only.
if (!handle.prepare("SELECT 1 FROM settings WHERE key='rcon_private_diagnostics_v1'").get()) {
if (
!handle.prepare("SELECT 1 FROM settings WHERE key='rcon_private_diagnostics_v1'").get()
) {
handle.transaction(() => {
handle.exec(`UPDATE servers SET last_reply_snippet=NULL;
UPDATE servers SET last_error='bad-reply:unparseable print reply'
@@ -374,25 +519,52 @@ export function db(): Database.Database {
migrateTable(handle, 'scripts', SCRIPTS_COLUMNS);
migrateTable(handle, 'script_versions', SCRIPT_VERSIONS_COLUMNS);
migrateTable(handle, 'beta_scripts', BETA_SCRIPTS_COLUMNS);
migrateTable(handle, 'beta_test_runs', [{ name: 'revision', ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1' }]);
migrateTable(handle, 'beta_test_runs', [
{
name: 'revision',
ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1'
}
]);
migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS);
migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS);
// Older databases gain rolled_back_by above, after schema.sql runs.
handle.exec('CREATE INDEX IF NOT EXISTS idx_deployments_rolled_back_by ON deployments(rolled_back_by)');
handle.exec(
'CREATE INDEX IF NOT EXISTS idx_deployments_rolled_back_by ON deployments(rolled_back_by)'
);
migrateTable(handle, 'server_launch_collections', [
{ name: 'cancelled_at', ddl: 'ALTER TABLE server_launch_collections ADD COLUMN cancelled_at TEXT' },
{ name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" }
{
name: 'cancelled_at',
ddl: 'ALTER TABLE server_launch_collections ADD COLUMN cancelled_at TEXT'
},
{
name: 'mode',
ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'"
},
{
name: 'retained_servers',
ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'"
}
]);
migrateTable(handle, 'server_launch_recipes', [
{ name: 'recipe_summary', ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT' }
{
name: 'recipe_summary',
ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT'
}
]);
migrateServerHistory(handle);
migrateTable(handle, 'server_collection_previews', [
{ name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'fleet_snapshot', ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT' }
{
name: 'mode',
ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'"
},
{
name: 'fleet_snapshot',
ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT'
}
]);
migrateTable(handle, 'server_telemetry', [
{ name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' }
]);
migrateTable(handle, 'server_telemetry', [{ name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' }]);
// Kind-referencing indexes: safe here because migrateTable above just
// ALTER'd the column into existence if it was missing.
handle.exec(
@@ -403,14 +575,27 @@ export function db(): Database.Database {
backfillMultiScriptJoins(handle);
assertSnapshotSchemeSafe(handle);
migrateGameTelemetry(handle);
migrateTable(handle, 'leaderboard_matches', [{ name: 'start_players', ddl: 'ALTER TABLE leaderboard_matches ADD COLUMN start_players INTEGER' }]);
migrateTable(handle, 'leaderboard_matches', [
{
name: 'start_players',
ddl: 'ALTER TABLE leaderboard_matches ADD COLUMN start_players INTEGER'
}
]);
migrateTable(handle, 'leaderboard_match_players', [
{ name: 'first_observed_round', ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN first_observed_round INTEGER' },
{ name: 'present_at_start', ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN present_at_start INTEGER' }
{
name: 'first_observed_round',
ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN first_observed_round INTEGER'
},
{
name: 'present_at_start',
ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN present_at_start INTEGER'
}
]);
backfillLeaderboard(handle);
initializeFleetSettings(handle);
migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
migrationStep('native instance schema and file plans', () =>
migrateNativeInstances(handle)
);
migrationStep('per-game server key pools', () => migrateServerKeys(handle));
migrationStep('autoscaling schema', () => ensureAutoscalingSchema(handle));
migrationStep('rebalancing schema', () => ensureRebalancingSchema(handle));
@@ -436,12 +621,18 @@ export function db(): Database.Database {
ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`);
installPerformanceIndexes(handle);
installReportRevisions(handle);
migrationStep('retention clocks and indexes', () => retentionClocks(handle, {
deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'],
maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'],
leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at']
}));
migrationStep('retention clocks and indexes', () =>
retentionClocks(handle, {
deployments: ['started_at', 'finished_at'],
server_deployments: ['started_at', 'finished_at'],
maintenance_jobs: ['requested_at', 'finished_at'],
watchdog_incidents: ['first_seen_at', 'finished_at'],
leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
betas: ['opened_at', 'closed_at'],
script_versions: ['uploaded_at'],
retention_controls: ['resolved_at']
})
);
// Existing databases require one rebuild to gain the incremental-vacuum page map.
enableIncrementalVacuum(handle);
handle.pragma('optimize = 0x10002');
@@ -472,8 +663,11 @@ export function db(): Database.Database {
_db = handle;
optimizeTimer = setInterval(() => {
try { handle.pragma('optimize'); }
catch (error) { console.error('[database] optimize failed:', error); }
try {
handle.pragma('optimize');
} catch (error) {
console.error('[database] optimize failed:', error);
}
}, 60 * 60_000);
optimizeTimer.unref();
return _db;
src/lib/server/db/retention-clocks.ts
+53 -15
View File
@@ -3,30 +3,59 @@ import { migrationStep, tuneMigrationConnection } from './connection.js';
/** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */
export function retentionClocks(handle: Database.Database, clocks: Record<string, string[]>): void {
handle.exec('CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)');
handle.exec(
'CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)'
);
for (const [table, columns] of Object.entries(clocks)) {
const marker = `iso-clocks:${table}:v1`;
if (!handle.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?').get(marker)) {
if (
!handle
.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?')
.get(marker)
) {
handle.transaction(() => {
for (const column of columns) handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column})
for (const column of columns)
handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column})
WHERE julianday(${column}) IS NOT NULL AND ${column}!=strftime('%Y-%m-%dT%H:%M:%fZ',${column});`);
handle.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)').run(marker);
handle
.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)')
.run(marker);
})();
}
const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as { name: string; partial: number }[];
const indexed = new Set(indexes.filter(index => !index.partial).map(index =>
(handle.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`).get() as { name: string }).name));
for (const column of columns) if (!indexed.has(column)) {
handle.exec(`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`);
}
const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as {
name: string;
partial: number;
}[];
const indexed = new Set(
indexes
.filter((index) => !index.partial)
.map(
(index) =>
(
handle
.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`)
.get() as { name: string }
).name
)
);
for (const column of columns)
if (!indexed.has(column)) {
handle.exec(
`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`
);
}
}
}
/** VACUUM can renumber implicit rowids. Keep host enable cutoffs tied to command IDs across crashes. */
export function enableIncrementalVacuum(handle: Database.Database): void {
if (handle.pragma('auto_vacuum', { simple: true }) !== 2) {
handle.exec(`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`);
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) {
handle.exec(
`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`
);
if (
handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()
) {
handle.exec(`INSERT OR IGNORE INTO database_vacuum_boundaries
SELECT e.host_id,(SELECT o.id FROM ec_host_operations o WHERE o.host_id=e.host_id
AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`);
@@ -42,7 +71,9 @@ export function enableIncrementalVacuum(handle: Database.Database): void {
};
try {
tuneMigrationConnection(handle);
migrationStep('incremental vacuum page map (disk-backed rebuild)', () => handle.exec('VACUUM'));
migrationStep('incremental vacuum page map (disk-backed rebuild)', () =>
handle.exec('VACUUM')
);
} finally {
handle.pragma('shrink_memory');
handle.pragma(`temp_store = ${previous.temp}`);
@@ -51,9 +82,16 @@ export function enableIncrementalVacuum(handle: Database.Database): void {
handle.pragma(`journal_size_limit = ${previous.journal}`);
}
}
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) {
if (
handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()
) {
handle.transaction(() => {
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) handle.exec(`
if (
handle
.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'")
.get()
)
handle.exec(`
UPDATE ec_host_enable_requests AS e SET operation_floor=COALESCE((
SELECT o.rowid FROM database_vacuum_boundaries b JOIN ec_host_operations o ON o.id=b.operation_id
WHERE b.host_id=e.host_id),0) WHERE e.host_id IN (SELECT host_id FROM database_vacuum_boundaries)`);
src/lib/server/db/server-history.ts
+47 -18
View File
@@ -1,17 +1,34 @@
import type Database from 'better-sqlite3';
const historyTables = ['beta_servers', 'deployment_servers', 'server_deployments', 'leaderboard_matches', 'watchdog_incidents', 'maintenance_jobs'];
const historyTables = [
'beta_servers',
'deployment_servers',
'server_deployments',
'leaderboard_matches',
'watchdog_incidents',
'maintenance_jobs'
];
const now = "strftime('%Y-%m-%dT%H:%M:%fZ','now')";
/** Detach durable beta records without dropping their evidence, indexes, or activity triggers. */
function detachBetaHistory(handle: Database.Database): void {
for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) {
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
const replacement = sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
.get(name) as { sql: string };
const replacement = sql.replace(
/server\s+TEXT NOT NULL REFERENCES servers\(name\)/,
'server TEXT NOT NULL'
);
if (replacement === sql) continue;
const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependents = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all(name) as { sql: string }[];
const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ');
const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[])
.map((col) => col.name)
.join(', ');
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
@@ -19,9 +36,12 @@ function detachBetaHistory(handle: Database.Database): void {
handle.exec(`INSERT INTO ${name}_history_new (rowid, ${columns}) SELECT rowid, ${columns} FROM ${name};
DROP TABLE ${name}; ALTER TABLE ${name}_history_new RENAME TO ${name};`);
for (const dependent of dependents) handle.exec(dependent.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Server history migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Server history migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
}
@@ -40,12 +60,19 @@ export function migrateServerHistory(handle: Database.Database): void {
detachBetaHistory(handle);
handle.transaction(() => {
for (const table of historyTables) {
const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[];
if (!columns.some(column => column.name === 'server_deleted_at')) handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`);
const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as {
name: string;
}[];
if (!columns.some((column) => column.name === 'server_deleted_at'))
handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`);
}
const collectionColumns = handle.prepare('PRAGMA table_info(server_launch_collections)').all() as { name: string }[];
if (!collectionColumns.some(column => column.name === 'retained_server_deletions')) {
handle.exec("ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'");
const collectionColumns = handle
.prepare('PRAGMA table_info(server_launch_collections)')
.all() as { name: string }[];
if (!collectionColumns.some((column) => column.name === 'retained_server_deletions')) {
handle.exec(
"ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'"
);
}
// Historical unfinished work cannot claim the next server with this name.
handle.exec(`DROP INDEX IF EXISTS idx_maintenance_pending;
@@ -96,15 +123,17 @@ export function migrateServerHistory(handle: Database.Database): void {
OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`;
for (const table of historyTables) {
const exception = table === 'server_deployments'
? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS (
const exception =
table === 'server_deployments'
? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS (
SELECT 1 FROM server_deployments removal WHERE removal.server = server_deployments.server
AND removal.rowid > server_deployments.rowid AND removal.action = 'destroy' AND removal.state IN ('done','warning')))
AND id NOT IN (${liveRebuilds})`
: table === 'deployment_servers'
? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))`
: table === 'watchdog_incidents'
? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))` : '';
: table === 'deployment_servers'
? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))`
: table === 'watchdog_incidents'
? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))`
: '';
handle.exec(`UPDATE ${table} SET server_deleted_at = ${now} WHERE server_deleted_at IS NULL
AND NOT EXISTS (SELECT 1 FROM servers WHERE name = ${table}.server AND destroyed_at IS NULL) ${exception}`);
}
src/lib/server/db/server-keys.ts
+116 -64
View File
@@ -22,79 +22,131 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_server_key_leases_current ON server_key_le
/** Preserve every occupied legacy assignment, even duplicated credentials. */
export function migrateServerKeys(handle: Database.Database): void {
handle.transaction(() => {
const legacy = (handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[])
.some(column => column.name === 'server_id');
if (legacy) handle.exec('ALTER TABLE server_keys RENAME TO legacy_server_keys');
handle.exec(keySchema + leaseSchema);
if (!legacy) return;
const rows = handle.prepare(`SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id`).all() as
{ server_id: string; pluto_key: string; updated_at: string }[];
const occupied = handle.prepare(`SELECT 1 WHERE
handle
.transaction(() => {
const legacy = (
handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[]
).some((column) => column.name === 'server_id');
if (legacy) handle.exec('ALTER TABLE server_keys RENAME TO legacy_server_keys');
handle.exec(keySchema + leaseSchema);
if (!legacy) return;
const rows = handle
.prepare(
`SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id`
)
.all() as { server_id: string; pluto_key: string; updated_at: string }[];
const occupied = handle.prepare(`SELECT 1 WHERE
EXISTS (SELECT 1 FROM servers WHERE name=@server AND destroyed_at IS NULL)
OR EXISTS (SELECT 1 FROM native_instances WHERE server=@server AND retired_at IS NULL)
OR EXISTS (SELECT 1 FROM server_deployments WHERE server=@server AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state NOT IN ('done','warning','cancelled'))`);
const find = handle.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?');
const serverGame = handle.prepare("SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')");
const insert = handle.prepare('INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)');
const lease = handle.prepare('INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)');
const assignments = new Map<string, string[]>();
function save(game: string, credential: string, label: string, updatedAt: string): string {
const existing = (find.get(game, credential) as { id: string } | undefined)?.id;
if (existing) return existing;
const id = randomUUID();
insert.run(id, game, label, credential, updatedAt);
return id;
}
for (const row of rows) {
const game = /^(t5|t6)_/.exec(row.server_id)?.[1]
?? (serverGame.get(row.server_id) as { game: string } | undefined)?.game;
if (!game) throw new Error(`Cannot determine the game for saved server key ${row.server_id}`);
const id = save(game, row.pluto_key, row.server_id, row.updated_at);
if (occupied.get({ server: row.server_id })) assignments.set(row.server_id, [id]);
}
// The old form could overwrite a saved key without changing the running
// container. Durable create/rebuild payloads are stronger evidence of
// its actual credential. Keep ambiguous delivered attempts as additional
// holds until the instance is destroyed; never free a possibly live key.
const hasTable = (name: string) => !!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name);
if (hasTable('ec_host_operations')) {
const payload = hasTable('ec_host_operation_payloads')
? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)' : 'o.payload';
const command = `CASE WHEN json_valid(${payload}) THEN ${payload} ELSE '{}' END`;
const launched = handle.prepare(`SELECT n.server,o.state,o.updated_at,
const find = handle.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?');
const serverGame = handle.prepare(
"SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')"
);
const insert = handle.prepare(
'INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)'
);
const lease = handle.prepare(
'INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)'
);
const assignments = new Map<string, string[]>();
function save(
game: string,
credential: string,
label: string,
updatedAt: string
): string {
const existing = (find.get(game, credential) as { id: string } | undefined)?.id;
if (existing) return existing;
const id = randomUUID();
insert.run(id, game, label, credential, updatedAt);
return id;
}
for (const row of rows) {
const game =
/^(t5|t6)_/.exec(row.server_id)?.[1] ??
(serverGame.get(row.server_id) as { game: string } | undefined)?.game;
if (!game)
throw new Error(
`Cannot determine the game for saved server key ${row.server_id}`
);
const id = save(game, row.pluto_key, row.server_id, row.updated_at);
if (occupied.get({ server: row.server_id })) assignments.set(row.server_id, [id]);
}
// The old form could overwrite a saved key without changing the running
// container. Durable create/rebuild payloads are stronger evidence of
// its actual credential. Keep ambiguous delivered attempts as additional
// holds until the instance is destroyed; never free a possibly live key.
const hasTable = (name: string) =>
!!handle
.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?")
.get(name);
if (hasTable('ec_host_operations')) {
const payload = hasTable('ec_host_operation_payloads')
? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'
: 'o.payload';
const command = `CASE WHEN json_valid(${payload}) THEN ${payload} ELSE '{}' END`;
const launched = handle
.prepare(
`SELECT n.server,o.state,o.updated_at,
json_extract(${command},'$.env.PLUTO_SERVER_KEY') AS credential,
json_extract(${command},'$.game') AS game,
json_extract(${command},'$.env.PLUTAINER_GAME') AS game_id
FROM native_instances n JOIN ec_host_operations o ON o.instance_id=n.id AND o.host_id=n.host_id
WHERE n.retired_at IS NULL AND o.kind IN ('instance.create','instance.rebuild') AND o.state!='failed'
AND (o.generation=n.generation OR json_extract(${command},'$.nextGeneration')=n.generation)
ORDER BY o.rowid DESC`).all() as
{ server: string; state: string; updated_at: string; credential: unknown; game: unknown; game_id: unknown }[];
const recovered = new Map<string, string[]>();
const complete = new Set<string>();
for (const row of launched) {
if (complete.has(row.server) || typeof row.credential !== 'string' || !row.credential) continue;
const game = row.game === 't5' || row.game === 't6' ? row.game
: row.game_id === 't5sp' ? 't5' : row.game_id === 't6zm' ? 't6'
: /^(t5|t6)_/.exec(row.server)?.[1]
?? (serverGame.get(row.server) as { game: string } | undefined)?.game;
if (!game) throw new Error(`Cannot determine the game for launched server ${row.server}`);
const ids = recovered.get(row.server) ?? [];
const id = save(game, row.credential, row.server, row.updated_at);
if (!ids.includes(id)) ids.push(id);
recovered.set(row.server, ids);
if (row.state === 'completed') complete.add(row.server);
ORDER BY o.rowid DESC`
)
.all() as {
server: string;
state: string;
updated_at: string;
credential: unknown;
game: unknown;
game_id: unknown;
}[];
const recovered = new Map<string, string[]>();
const complete = new Set<string>();
for (const row of launched) {
if (
complete.has(row.server) ||
typeof row.credential !== 'string' ||
!row.credential
)
continue;
const game =
row.game === 't5' || row.game === 't6'
? row.game
: row.game_id === 't5sp'
? 't5'
: row.game_id === 't6zm'
? 't6'
: (/^(t5|t6)_/.exec(row.server)?.[1] ??
(serverGame.get(row.server) as { game: string } | undefined)
?.game);
if (!game)
throw new Error(
`Cannot determine the game for launched server ${row.server}`
);
const ids = recovered.get(row.server) ?? [];
const id = save(game, row.credential, row.server, row.updated_at);
if (!ids.includes(id)) ids.push(id);
recovered.set(row.server, ids);
if (row.state === 'completed') complete.add(row.server);
}
for (const [server, ids] of recovered) {
// If no completed launch receipt survives, its saved assignment
// remains a possible prior credential while the attempt resolves.
if (!complete.has(server))
for (const id of assignments.get(server) ?? [])
if (!ids.includes(id)) ids.push(id);
assignments.set(server, ids);
}
}
for (const [server, ids] of recovered) {
// If no completed launch receipt survives, its saved assignment
// remains a possible prior credential while the attempt resolves.
if (!complete.has(server)) for (const id of assignments.get(server) ?? []) if (!ids.includes(id)) ids.push(id);
assignments.set(server, ids);
}
}
for (const [server, ids] of assignments) ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0));
handle.exec('DROP TABLE legacy_server_keys');
}).immediate();
for (const [server, ids] of assignments)
ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0));
handle.exec('DROP TABLE legacy_server_keys');
})
.immediate();
}
src/lib/server/db/slow-statements.ts
+21 -9
View File
@@ -1,19 +1,31 @@
import type Database from 'better-sqlite3';
/** Log query shapes, never bound values or SQL string literals. Disabled by default. */
export function timeStatement(statement: Database.Statement, sql: string, thresholdMs: number): Database.Statement {
export function timeStatement(
statement: Database.Statement,
sql: string,
thresholdMs: number
): Database.Statement {
if (!Number.isFinite(thresholdMs) || thresholdMs <= 0) return statement;
const shape = sql.replace(/'(?:''|[^'])*'/g, '?').replace(/\s+/g, ' ').trim().slice(0, 200);
const shape = sql
.replace(/'(?:''|[^'])*'/g, '?')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 200);
for (const method of ['run', 'get', 'all'] as const) {
const original = statement[method].bind(statement) as (...args: unknown[]) => unknown;
Object.defineProperty(statement, method, { value: (...args: unknown[]) => {
const started = performance.now();
try { return original(...args); }
finally {
const ms = performance.now() - started;
if (ms >= thresholdMs) console.warn('[sql]', { ms: Math.round(ms), method, sql: shape });
Object.defineProperty(statement, method, {
value: (...args: unknown[]) => {
const started = performance.now();
try {
return original(...args);
} finally {
const ms = performance.now() - started;
if (ms >= thresholdMs)
console.warn('[sql]', { ms: Math.round(ms), method, sql: shape });
}
}
} });
});
}
return statement;
}
src/lib/server/db/version-layout.ts
+27 -9
View File
@@ -3,25 +3,43 @@ import type Database from 'better-sqlite3';
/** Keep metadata ahead of overflow pages, preserving IDs and all references. */
export function migrateVersionLayout(handle: Database.Database): void {
const columns = handle.pragma('table_info(script_versions)') as { name: string }[];
if (!columns.some(column => column.name === 'content') || columns.at(-1)?.name === 'content') return;
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'").get() as { sql: string };
if (!columns.some((column) => column.name === 'content') || columns.at(-1)?.name === 'content')
return;
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'")
.get() as { sql: string };
const content = /\bcontent\s+BLOB\s+NOT\s+NULL\s*,/i;
if (!content.test(sql)) throw new Error('Unrecognized script version content layout');
const replacement = sql.replace(content, '').replace(/,\s*UNIQUE\s*\(/i, ', content BLOB NOT NULL, UNIQUE(')
const replacement = sql
.replace(content, '')
.replace(/,\s*UNIQUE\s*\(/i, ', content BLOB NOT NULL, UNIQUE(')
.replace('script_versions', 'script_versions_metadata');
const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='script_versions' AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependents = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name='script_versions' AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all() as { sql: string }[];
const sequence = handle.prepare("SELECT seq FROM sqlite_sequence WHERE name='script_versions'").get() as { seq: number } | undefined;
const sequence = handle
.prepare("SELECT seq FROM sqlite_sequence WHERE name='script_versions'")
.get() as { seq: number } | undefined;
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement);
const names = columns.map(column => `"${column.name}"`).join(',');
const names = columns.map((column) => `"${column.name}"`).join(',');
handle.exec(`INSERT INTO script_versions_metadata (${names}) SELECT ${names} FROM script_versions;
DROP TABLE script_versions; ALTER TABLE script_versions_metadata RENAME TO script_versions;`);
if (sequence) handle.prepare("UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name='script_versions'").run(sequence.seq);
if (sequence)
handle
.prepare(
"UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name='script_versions'"
)
.run(sequence.seq);
for (const dependent of dependents) handle.exec(dependent.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Version layout migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Version layout migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
src/lib/server/fleet/beta-evaluation.ts
+81 -26
View File
@@ -19,14 +19,30 @@ export interface TestMetrics {
outages: number;
streaks: Record<string, number>;
}
export const emptyMetrics = (): TestMetrics => ({ values: {}, manual: {}, outages: 0, streaks: {} });
export const emptyMetrics = (): TestMetrics => ({
values: {},
manual: {},
outages: 0,
streaks: {}
});
/** Gameplay comes only from the script. No reporter IDs or beta handshake. */
export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: GameEvent, now: number): void {
export function applyGameEvent(
plan: BetaPlan,
metrics: TestMetrics,
event: GameEvent,
now: number
): void {
const players = event.players.length;
if (event.event === 'game_start' || !metrics.match || metrics.match.map !== event.map) {
metrics.match = { map: event.map, started: event.event === 'game_start' ? now : undefined,
lastEvent: now, players, ended: false, rounds: {} };
metrics.match = {
map: event.map,
started: event.event === 'game_start' ? now : undefined,
lastEvent: now,
players,
ended: false,
rounds: {}
};
metrics.streaks = {};
}
const match = metrics.match;
@@ -34,10 +50,12 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game
const minutes = Math.max(0, now - match.lastEvent) / 60_000;
for (const r of plan.requirements) {
const current = metrics.values[r.id] ?? 0;
const populated = players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers);
const populated =
players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers);
if (r.kind === 'population') metrics.values[r.id] = Math.max(current, players);
if (r.kind === 'populated-minutes' && populated) metrics.values[r.id] = current + minutes;
if (r.kind === 'player-hours' && populated) metrics.values[r.id] = current + minutes * Math.min(players, match.players) / 60;
if (r.kind === 'player-hours' && populated)
metrics.values[r.id] = current + (minutes * Math.min(players, match.players)) / 60;
if (r.kind === 'sustained-population') {
metrics.streaks[r.id] = populated ? (metrics.streaks[r.id] ?? 0) + minutes : 0;
metrics.values[r.id] = Math.max(current, metrics.streaks[r.id]);
@@ -46,10 +64,14 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game
match.rounds[r.id] = Math.max(match.rounds[r.id] ?? 0, event.round);
if (r.kind === 'round') metrics.values[r.id] = Math.max(current, event.round);
}
if (event.event === 'game_end' && r.kind === 'matches'
&& event.round >= MIN_RECORDED_ROUND
&& (match.rounds[r.id] ?? -1) >= r.minRound
&& (r.minMinutes === 0 || (match.started !== undefined && now - match.started >= r.minMinutes * 60_000))) {
if (
event.event === 'game_end' &&
r.kind === 'matches' &&
event.round >= MIN_RECORDED_ROUND &&
(match.rounds[r.id] ?? -1) >= r.minRound &&
(r.minMinutes === 0 ||
(match.started !== undefined && now - match.started >= r.minMinutes * 60_000))
) {
metrics.values[r.id] = current + 1;
}
}
@@ -66,30 +88,63 @@ export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy:
const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
if (!healthy && m.wasHealthy === true) m.outages++;
for (const r of plan.requirements) {
if (r.kind === 'healthy-minutes') m.values[r.id] = healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0;
if (r.kind === 'healthy-minutes')
m.values[r.id] =
healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0;
}
m.lastSample = at; m.wasHealthy = healthy;
m.lastSample = at;
m.wasHealthy = healthy;
}
export function evaluateRequirements(plan: BetaPlan, metrics: TestMetrics, phase: number): RequirementProgress[] {
return plan.requirements.map(r => {
export function evaluateRequirements(
plan: BetaPlan,
metrics: TestMetrics,
phase: number
): RequirementProgress[] {
return plan.requirements.map((r) => {
const budget = r.kind === 'failure-budget';
const current = budget ? metrics.outages : r.kind === 'manual' ? Number(!!metrics.manual[r.id])
: r.kind === 'recovery' ? phase : metrics.values[r.id] ?? 0;
return { id: r.id, label: r.label, current, target: r.target, passed: budget ? current <= r.target : current >= r.target,
const current = budget
? metrics.outages
: r.kind === 'manual'
? Number(!!metrics.manual[r.id])
: r.kind === 'recovery'
? phase
: (metrics.values[r.id] ?? 0);
return {
id: r.id,
label: r.label,
current,
target: r.target,
passed: budget ? current <= r.target : current >= r.target,
failed: budget && current > r.target,
detail: r.kind === 'manual' ? metrics.manual[r.id] ?? 'Awaiting tester sign-off'
: budget ? `${current} / ${r.target} allowed; monitored until promotion`
: `${Math.round(current * 100) / 100} / ${r.target}` };
detail:
r.kind === 'manual'
? (metrics.manual[r.id] ?? 'Awaiting tester sign-off')
: budget
? `${current} / ${r.target} allowed; monitored until promotion`
: `${Math.round(current * 100) / 100} / ${r.target}`
};
});
}
export function coveragePassed(plan: BetaPlan, servers: Array<{ passed: boolean; game: string | null; map: string | null }>): boolean {
export function coveragePassed(
plan: BetaPlan,
servers: Array<{ passed: boolean; game: string | null; map: string | null }>
): boolean {
if (!servers.length) return false;
if (plan.scope === 'all') return servers.every(s => s.passed);
if (plan.scope === 'quorum') return servers.filter(s => s.passed).length >= plan.quorum;
const key = (s: typeof servers[number]) => plan.scope === 'game' ? s.game : plan.scope === 'map' ? s.map
: s.game && s.map ? `${s.game}/${s.map}` : null;
if (plan.scope === 'all') return servers.every((s) => s.passed);
if (plan.scope === 'quorum') return servers.filter((s) => s.passed).length >= plan.quorum;
const key = (s: (typeof servers)[number]) =>
plan.scope === 'game'
? s.game
: plan.scope === 'map'
? s.map
: s.game && s.map
? `${s.game}/${s.map}`
: null;
const groups = new Set(servers.map(key));
return !groups.has(null) && [...groups].every(group => servers.some(s => s.passed && key(s) === group));
return (
!groups.has(null) &&
[...groups].every((group) => servers.some((s) => s.passed && key(s) === group))
);
}
src/lib/server/fleet/beta-pages.ts
+39 -12
View File
@@ -8,13 +8,21 @@ const positive = (value: string | null, fallback: number) => {
};
export function betaListPage(params: URLSearchParams) {
const state = params.get('state'), view = params.get('view'), search = (params.get('q') ?? '').slice(0, 200).trim();
const predicates: string[] = [], args: (string | number)[] = [];
if (state) { predicates.push('b.state=?'); args.push(state); }
else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')");
const state = params.get('state'),
view = params.get('view'),
search = (params.get('q') ?? '').slice(0, 200).trim();
const predicates: string[] = [],
args: (string | number)[] = [];
if (state) {
predicates.push('b.state=?');
args.push(state);
} else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')");
else if (view === 'history') predicates.push("b.state IN ('promoted','rolled-back')");
const status = params.get('status');
if (status && ['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status)) {
if (
status &&
['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status)
) {
// Match betaStatus: closed lifecycle wins over the final test outcome.
predicates.push(`(CASE WHEN b.state IN ('promoted','rolled-back') THEN b.state
WHEN EXISTS (SELECT 1 FROM beta_servers m WHERE m.beta_id=b.id AND (m.server_deleted_at IS NOT NULL OR m.stage='failed'))
@@ -24,8 +32,15 @@ export function betaListPage(params: URLSearchParams) {
args.push(status);
}
if (search) {
const names = (db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[])
.filter(row => `${row.server} ${serverTitle(row.server)}`.toLowerCase().includes(search.toLowerCase())).map(row => row.server);
const names = (
db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[]
)
.filter((row) =>
`${row.server} ${serverTitle(row.server)}`
.toLowerCase()
.includes(search.toLowerCase())
)
.map((row) => row.server);
predicates.push(`(instr(lower('#'||b.id||' '||COALESCE(b.notes,'')),lower(?))>0 OR EXISTS (
SELECT 1 FROM beta_scripts bs JOIN scripts s ON s.id=bs.script_id WHERE bs.beta_id=b.id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0))
OR EXISTS (SELECT 1 FROM scripts s WHERE s.id=b.script_id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0))
@@ -33,21 +48,33 @@ export function betaListPage(params: URLSearchParams) {
args.push(search, search, search, search, search, JSON.stringify(names));
}
const where = predicates.length ? ' WHERE ' + predicates.join(' AND ') : '';
const total = (db().prepare('SELECT COUNT(*) n FROM betas b' + where).get(...args) as { n: number }).n;
const total = (
db()
.prepare('SELECT COUNT(*) n FROM betas b' + where)
.get(...args) as { n: number }
).n;
const pageSize = Math.min(100, positive(params.get('limit'), 25));
const totalPages = Math.max(1, Math.ceil(total / pageSize));
const page = Math.min(totalPages, positive(params.get('page'), 1));
const betas = db().prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?')
const betas = db()
.prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?')
.all(...args, pageSize, (page - 1) * pageSize) as BetaRow[];
const counts = db().prepare(`SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active,
COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas`).get() as { active: number; history: number };
const counts = db()
.prepare(
`SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active,
COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas`
)
.get() as { active: number; history: number };
return { betas, page, pageSize, total, totalPages, counts };
}
export function betaActivityPage(id: number, params = new URLSearchParams()) {
const before = positive(params.get('before'), Number.MAX_SAFE_INTEGER);
const limit = Math.min(100, positive(params.get('limit'), 50));
const rows = db().prepare('SELECT id,created_at,message FROM beta_events WHERE beta_id=? AND id<? ORDER BY id DESC LIMIT ?')
const rows = db()
.prepare(
'SELECT id,created_at,message FROM beta_events WHERE beta_id=? AND id<? ORDER BY id DESC LIMIT ?'
)
.all(id, before, limit + 1) as { id: number; created_at: string; message: string }[];
const more = rows.length > limit;
const events = rows.slice(0, limit).reverse();
src/lib/server/fleet/beta-revisions.ts
+51 -13
View File
@@ -2,29 +2,67 @@ import { db } from '../db/index.js';
import type { BetaRevision } from '../../beta-revision.js';
export function currentBetaRevision(id: number): { revision: number; started_at: string } {
return db().prepare(`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`).get(id) as
{ revision: number; started_at: string } | undefined
?? { revision: 1, started_at: (db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as { opened_at: string }).opened_at };
return (
(db()
.prepare(
`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`
)
.get(id) as { revision: number; started_at: string } | undefined) ?? {
revision: 1,
started_at: (
db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as {
opened_at: string;
}
).opened_at
}
);
}
/** Called in the revision transaction, before replacing the pinned bundle. */
export function archiveBetaRevision(id: number, revision: number, startedAt: string, notes: string | null): void {
db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)')
export function archiveBetaRevision(
id: number,
revision: number,
startedAt: string,
notes: string | null
): void {
db()
.prepare(
'INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)'
)
.run(id, revision, startedAt, notes);
db().prepare(`INSERT OR IGNORE INTO beta_revision_files
db()
.prepare(
`INSERT OR IGNORE INTO beta_revision_files
(beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content_hash)
SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content_hash
FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id
WHERE bs.beta_id = ?`).run(revision, id);
WHERE bs.beta_id = ?`
)
.run(revision, id);
}
export function betaRevisionHistory(id: number): BetaRevision[] {
const rows = db().prepare('SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC')
.all(id) as Array<{ revision: number; started_at: string; notes: string | null; results_json: string | null }>;
const files = db().prepare(`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`).all(id) as Array<BetaRevision['files'][number] & { revision: number }>;
return rows.map(({ results_json, ...row }) => ({ ...row,
const rows = db()
.prepare(
'SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC'
)
.all(id) as Array<{
revision: number;
started_at: string;
notes: string | null;
results_json: string | null;
}>;
const files = db()
.prepare(
`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`
)
.all(id) as Array<BetaRevision['files'][number] & { revision: number }>;
return rows.map(({ results_json, ...row }) => ({
...row,
results: results_json ? JSON.parse(results_json) : null,
files: files.filter(file => file.revision === row.revision).map(({ revision, ...file }) => file)
files: files
.filter((file) => file.revision === row.revision)
.map(({ revision, ...file }) => file)
}));
}
src/lib/server/fleet/beta-selection.ts
+103 -40
View File
@@ -7,7 +7,7 @@ import { observationReader } from './status.js';
import type { ScriptRow } from './scripts.js';
const games = ['t5', 't6'] as const;
type Game = typeof games[number];
type Game = (typeof games)[number];
type Counts = Partial<Record<Game, number>>;
export function parseBetaInstanceCounts(value: unknown): Counts {
@@ -16,7 +16,12 @@ export function parseBetaInstanceCounts(value: unknown): Counts {
}
const counts: Counts = {};
for (const [game, count] of Object.entries(value)) {
if (!games.includes(game as Game) || !Number.isSafeInteger(count) || count < 1 || count > 1000) {
if (
!games.includes(game as Game) ||
!Number.isSafeInteger(count) ||
count < 1 ||
count > 1000
) {
throw new Error('Instance counts must be integers from 1 to 1000 for t5 or t6');
}
counts[game as Game] = count;
@@ -25,79 +30,137 @@ export function parseBetaInstanceCounts(value: unknown): Counts {
}
function matches(script: ScriptRow, server: ServerRow): boolean {
return !script.scope_type || !script.scope_value
|| (script.scope_type === 'game' ? server.game : server.map) === script.scope_value;
return (
!script.scope_type ||
!script.scope_value ||
(script.scope_type === 'game' ? server.game : server.map) === script.scope_value
);
}
/** Choose at submission time using the same fresh observations shown in Fleet. */
export function selectBetaServers(scripts: ScriptRow[], instancesPerGame: unknown): string[] {
const counts = parseBetaInstanceCounts(instancesPerGame), observe = observationReader(), now = Date.now();
const hosts = new Map(listHosts().map(host => [host.id, host]));
const busy = new Set((db().prepare(`SELECT server FROM server_deployments WHERE server_deleted_at IS NULL
const counts = parseBetaInstanceCounts(instancesPerGame),
observe = observationReader(),
now = Date.now();
const hosts = new Map(listHosts().map((host) => [host.id, host]));
const busy = new Set(
(
db()
.prepare(
`SELECT server FROM server_deployments WHERE server_deleted_at IS NULL
AND (state IN ('queued','waiting','running','awaiting-health','interrupted')
OR (action IN ('destroy','rebuild') AND state = 'failed'
AND stage IN ('Stopping and removing container','Removing server files and registry')))`)
.all() as { server: string }[]).map(row => row.server));
const candidates = listServers().flatMap(server => {
if (!counts[server.game as Game] || server.desired_state === 'stopped' || busy.has(server.name)
|| !scripts.some(script => matches(script, server)) || protectedWork(server.name)) return [];
const host = server.host_id ? hosts.get(server.host_id) : null;
if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission)) return [];
const observation = observe(server);
if (observation.health !== 'online' || observation.last_players === null
|| !freshStatusTimestamp(observation.last_seen_at, now)) return [];
// An occupied RCON observation must not rank as empty just because a
// competing telemetry report claims zero players.
const rconPlayers = freshStatusTimestamp(server.last_seen_at, now)
&& (!observation.telemetry_after || Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after))
? server.last_players ?? 0 : 0;
return [{ server, players: Math.max(observation.last_players, rconPlayers) }];
}).sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name));
AND stage IN ('Stopping and removing container','Removing server files and registry')))`
)
.all() as { server: string }[]
).map((row) => row.server)
);
const candidates = listServers()
.flatMap((server) => {
if (
!counts[server.game as Game] ||
server.desired_state === 'stopped' ||
busy.has(server.name) ||
!scripts.some((script) => matches(script, server)) ||
protectedWork(server.name)
)
return [];
const host = server.host_id ? hosts.get(server.host_id) : null;
if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission))
return [];
const observation = observe(server);
if (
observation.health !== 'online' ||
observation.last_players === null ||
!freshStatusTimestamp(observation.last_seen_at, now)
)
return [];
// An occupied RCON observation must not rank as empty just because a
// competing telemetry report claims zero players.
const rconPlayers =
freshStatusTimestamp(server.last_seen_at, now) &&
(!observation.telemetry_after ||
Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after))
? (server.last_players ?? 0)
: 0;
return [{ server, players: Math.max(observation.last_players, rconPlayers) }];
})
.sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name));
for (const game of games) {
const available = candidates.filter(candidate => candidate.server.game === game).length;
const available = candidates.filter((candidate) => candidate.server.game === game).length;
if ((counts[game] ?? 0) > available) {
throw new Error(`${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.`);
throw new Error(
`${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.`
);
}
}
for (const script of scripts) {
if (!candidates.some(candidate => matches(script, candidate.server))) {
throw new Error(`${script.name}: no available server in the requested games matches this file's scope.`);
if (!candidates.some((candidate) => matches(script, candidate.server))) {
throw new Error(
`${script.name}: no available server in the requested games matches this file's scope.`
);
}
}
// Map scopes need a representative even when the least populated servers
// all run the same map. Reserve enough remaining slots for every map.
const requiredMaps = new Set(scripts.filter(script => script.scope_type === 'map').map(script => script.scope_value!));
const requiredMaps = new Set(
scripts.filter((script) => script.scope_type === 'map').map((script) => script.scope_value!)
);
function canComplete(remaining: typeof candidates, slots: Counts, maps: Set<string>): boolean {
const options = new Map<string, Set<Game>>();
for (const game of games) {
if (remaining.filter(candidate => candidate.server.game === game).length < (slots[game] ?? 0)) return false;
if (
remaining.filter((candidate) => candidate.server.game === game).length <
(slots[game] ?? 0)
)
return false;
}
for (const map of maps) {
const choices = new Set(remaining.filter(candidate => candidate.server.map === map && (slots[candidate.server.game as Game] ?? 0) > 0)
.map(candidate => candidate.server.game as Game));
const choices = new Set(
remaining
.filter(
(candidate) =>
candidate.server.map === map &&
(slots[candidate.server.game as Game] ?? 0) > 0
)
.map((candidate) => candidate.server.game as Game)
);
if (!choices.size) return false;
options.set(map, choices);
}
// There are two supported games: single-game maps reserve that game's
// slots, and maps available in either game use any remaining slot.
return games.every(game => [...options.values()].filter(choices => choices.size === 1 && choices.has(game)).length <= (slots[game] ?? 0))
&& maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0);
return (
games.every(
(game) =>
[...options.values()].filter(
(choices) => choices.size === 1 && choices.has(game)
).length <= (slots[game] ?? 0)
) && maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0)
);
}
if (!canComplete(candidates, counts, requiredMaps)) {
throw new Error('Increase the instance counts to cover every selected file\'s map scope, or choose servers in advanced options.');
throw new Error(
"Increase the instance counts to cover every selected file's map scope, or choose servers in advanced options."
);
}
const selected: string[] = [], slots = { ...counts };
const selected: string[] = [],
slots = { ...counts };
let maps = requiredMaps;
for (let index = 0; index < candidates.length; index++) {
const { server } = candidates[index], game = server.game as Game;
const { server } = candidates[index],
game = server.game as Game;
if (!slots[game]) continue;
const nextSlots = { ...slots, [game]: slots[game]! - 1 }, nextMaps = new Set(maps);
const nextSlots = { ...slots, [game]: slots[game]! - 1 },
nextMaps = new Set(maps);
if (server.map) nextMaps.delete(server.map);
if (!canComplete(candidates.slice(index + 1), nextSlots, nextMaps)) continue;
selected.push(server.name); slots[game] = nextSlots[game]; maps = nextMaps;
if (games.every(game => !slots[game])) break;
selected.push(server.name);
slots[game] = nextSlots[game];
maps = nextMaps;
if (games.every((game) => !slots[game])) break;
}
return selected;
}
src/lib/server/fleet/beta-tests.ts
+145 -36
View File
@@ -4,52 +4,120 @@ import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
import type { GameTelemetry } from '../../telemetry.js';
import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
import {
applySample,
applyGameEvent,
emptyMetrics,
evaluateRequirements,
type TestMetrics
} from './beta-evaluation.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { currentBetaRevision } from './beta-revisions.js';
import { invalidateBeta } from '../pipeline/bus.js';
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
export interface TestRun {
id: string;
beta_id: number;
server: string;
phase: number;
created_at: string;
observing_at: string | null;
metrics: string;
}
export interface StoredPlan {
beta_id: number;
plan: string;
outcome: string;
reason: string | null;
}
const nowIso = () => new Date().toISOString();
const activeParticipant = (id: number, server: string) => !!db().prepare(`SELECT 1 FROM beta_servers
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`).get(id, server);
export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined;
export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined;
export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); invalidateBeta(id); }
const activeParticipant = (id: number, server: string) =>
!!db()
.prepare(
`SELECT 1 FROM beta_servers
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
)
.get(id, server);
export const storedPlan = (id: number) =>
db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as
StoredPlan | undefined;
export const currentRun = (id: number, server: string) =>
db()
.prepare(
'SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL'
)
.get(id, server) as TestRun | undefined;
export function activity(id: number, message: string) {
db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message);
invalidateBeta(id);
}
export function setOutcome(id: number, outcome: string, reason: string | null): void {
const previous = storedPlan(id);
if (!previous || (previous.outcome === outcome && previous.reason === reason)) return;
db().prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?').run(outcome, reason, id);
db()
.prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?')
.run(outcome, reason, id);
activity(id, `Test plan ${outcome}${reason ? `: ${reason}` : ''}`);
}
export function readMetrics(run: TestRun): TestMetrics { return { ...emptyMetrics(), ...JSON.parse(run.metrics) }; }
export function saveMetrics(run: TestRun, metrics: TestMetrics): void { db().prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?').run(JSON.stringify(metrics), run.id); invalidateBeta(run.beta_id); }
export function readMetrics(run: TestRun): TestMetrics {
return { ...emptyMetrics(), ...JSON.parse(run.metrics) };
}
export function saveMetrics(run: TestRun, metrics: TestMetrics): void {
db()
.prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?')
.run(JSON.stringify(metrics), run.id);
invalidateBeta(run.beta_id);
}
export function evidence(run: TestRun, eventId: string, kind: string, payload: unknown): boolean {
return db().prepare('INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)')
.run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0;
return (
db()
.prepare(
'INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)'
)
.run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0
);
}
/** Internal progress record; never sent to or required from the game script. */
export async function prepareRun(betaId: number, server: string, phase = 0): Promise<TestRun> {
if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta');
if (!activeParticipant(betaId, server))
throw new Error('Original test server was deleted; start a new beta');
db().transaction(() => {
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server);
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), currentBetaRevision(betaId).revision);
db()
.prepare(
'UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL'
)
.run(nowIso(), betaId, server);
db()
.prepare(
'INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)'
)
.run(
randomUUID(),
betaId,
server,
phase,
nowIso(),
JSON.stringify(emptyMetrics()),
currentBetaRevision(betaId).revision
);
})();
return currentRun(betaId, server)!;
}
/** Route ordinary server events to every deployed, active beta automatically. */
export function recordGameEvent(event: GameTelemetry): void {
const runs = db().prepare(`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
const runs = db()
.prepare(
`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
JOIN beta_test_plans p ON p.beta_id = b.id
JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server
WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote')
AND s.server_deleted_at IS NULL
AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready')
AND s.restarted_at IS NOT NULL`).all(event.server) as TestRun[];
AND s.restarted_at IS NOT NULL`
)
.all(event.server) as TestRun[];
for (const run of runs) {
const plan = parseBetaPlan(JSON.parse(storedPlan(run.beta_id)!.plan));
const metrics = readMetrics(run);
@@ -64,17 +132,27 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
const m = readMetrics(run);
const row = getServer(run.server);
const server = row ? serverObservation(row) : null;
const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
const timestamp =
server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
const sample = Date.parse(timestamp ?? '');
const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
&& Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS;
const freshSample =
Number.isFinite(sample) &&
sample >= Date.parse(run.observing_at ?? run.created_at) &&
Date.now() - sample >= 0 &&
Date.now() - sample <= ONLINE_MAX_MS;
const healthy = freshSample && server?.health === 'online';
const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
const before = m.wasHealthy;
const lastSample = m.lastSample;
applySample(plan, m, at, healthy);
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, source: server?.status_source });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
if (freshSample && sample !== lastSample)
evidence(run, `sample:${sample}`, 'status-sample', {
healthy,
checkedAt: timestamp,
source: server?.status_source
});
if (before !== m.wasHealthy)
evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
saveMetrics(run, m);
}
@@ -82,22 +160,53 @@ export function testView(betaId: number): BetaTestView | null {
const stored = storedPlan(betaId);
if (!stored) return null;
const plan = parseBetaPlan(JSON.parse(stored.plan));
const runs = db().prepare(`SELECT r.*, s.server_deleted_at FROM beta_test_runs r
const runs = db()
.prepare(
`SELECT r.*, s.server_deleted_at FROM beta_test_runs r
JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`).all(betaId) as Array<TestRun & { server_deleted_at: string | null }>;
return { plan, outcome: stored.outcome, reason: stored.reason,
servers: runs.map(run => {
WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`
)
.all(betaId) as Array<TestRun & { server_deleted_at: string | null }>;
return {
plan,
outcome: stored.outcome,
reason: stored.reason,
servers: runs.map((run) => {
const metrics = readMetrics(run);
const progress = evaluateRequirements(plan, metrics, run.phase);
const latest = db().prepare("SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1").get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
const latest = db()
.prepare(
"SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1"
)
.get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
const event = latest ? JSON.parse(latest.payload) : null;
return { server: run.server, server_deleted_at: run.server_deleted_at, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at,
observation: { lastGameplayAt: latest?.received_at ?? null, lastEvent: latest?.kind ?? null,
players: metrics.match?.players ?? null, round: event?.round ?? null,
matchStarted: metrics.match?.started !== undefined, gameEnded: metrics.match?.ended ?? false,
healthy: metrics.wasHealthy ?? null, healthCheckedAt: metrics.lastSample ? new Date(metrics.lastSample).toISOString() : null } };
return {
server: run.server,
server_deleted_at: run.server_deleted_at,
phase: run.phase,
passed: progress.every((r) => r.passed),
progress,
observingAt: run.observing_at,
observation: {
lastGameplayAt: latest?.received_at ?? null,
lastEvent: latest?.kind ?? null,
players: metrics.match?.players ?? null,
round: event?.round ?? null,
matchStarted: metrics.match?.started !== undefined,
gameEnded: metrics.match?.ended ?? false,
healthy: metrics.wasHealthy ?? null,
healthCheckedAt: metrics.lastSample
? new Date(metrics.lastSample).toISOString()
: null
}
};
}),
evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
evidence: db()
.prepare(
`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`).all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence'] };
WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`
)
.all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence']
};
}
src/lib/server/fleet/beta.ts
+783 -251
View File
File diff suppressed because it is too large Load Diff
src/lib/server/fleet/chat.ts
+184 -49
View File
@@ -1,32 +1,61 @@
import { createHash } from 'node:crypto';
import { db } from '../db/index.js';
import { enrichGameChatTimeline, type ChatMatch, type ChatRoundReport, type ContextualChatEntry } from '../../game-chat.js';
import {
enrichGameChatTimeline,
type ChatMatch,
type ChatRoundReport,
type ContextualChatEntry
} from '../../game-chat.js';
import { parseGameEvent } from '../../telemetry.js';
import { telemetryRetention, telemetryCutoff } from './telemetry-requests.js';
import { getServer, listServers, type ServerRow } from './servers.js';
import { gameLogKey, resolveLogPath, readLogChunk, type LogCursor, type TailResult } from './logs.js';
import {
gameLogKey,
resolveLogPath,
readLogChunk,
type LogCursor,
type TailResult
} from './logs.js';
import { listHosts } from '../hosts/index.js';
import { observationReader } from './status.js';
/** Historical context is scoped to this registration, including round-one matches. */
export function serverChatTimeline(server: ServerRow, tail: TailResult): ContextualChatEntry[] {
const unscoped = enrichGameChatTimeline(tail.lines, { mtime: tail.mtime, matches: [], reports: [] });
const times = unscoped.flatMap(entry => entry.context.realTime ? [entry.context.realTime] : []);
const unscoped = enrichGameChatTimeline(tail.lines, {
mtime: tail.mtime,
matches: [],
reports: []
});
const times = unscoped.flatMap((entry) =>
entry.context.realTime ? [entry.context.realTime] : []
);
if (!times.length) return unscoped;
const since = times.reduce((a, b) => a < b ? a : b), until = times.reduce((a, b) => a > b ? a : b);
const since = times.reduce((a, b) => (a < b ? a : b)),
until = times.reduce((a, b) => (a > b ? a : b));
const handle = db();
const matches = handle.prepare(`SELECT id, started_at AS startedAt, ended_at AS endedAt,
const matches = handle
.prepare(
`SELECT id, started_at AS startedAt, ended_at AS endedAt,
last_received_at AS lastReceivedAt, status, start_observed AS startObserved
FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL
AND started_at <= ? AND (COALESCE(ended_at,last_received_at) >= ? OR status='active')
ORDER BY id DESC LIMIT 2000`).all(server.name, until, since) as Array<Omit<ChatMatch, 'startObserved'> & { startObserved: number }>;
ORDER BY id DESC LIMIT 2000`
)
.all(server.name, until, since) as Array<
Omit<ChatMatch, 'startObserved'> & { startObserved: number }
>;
// Diagnostic history survives deletion. Its receipt time must belong to the
// current registration so a reused server name cannot inherit old rounds.
const cutoff = telemetryCutoff();
const roundSince = new Date(Date.parse(since) - 120_000).toISOString();
const matchSince = matches.reduce((earliest, match) => match.startedAt < earliest ? match.startedAt : earliest, since);
const reports = handle.prepare(`SELECT received_at AS receivedAt, event, round,
const matchSince = matches.reduce(
(earliest, match) => (match.startedAt < earliest ? match.startedAt : earliest),
since
);
const reports = handle
.prepare(
`SELECT received_at AS receivedAt, event, round,
player_count AS playerCount FROM telemetry_requests
WHERE server = ? AND status = 200 AND received_at >= ?
AND julianday(received_at) >= julianday(?)
@@ -34,36 +63,69 @@ export function serverChatTimeline(server: ServerRow, tail: TailResult): Context
AND received_at <= ?
AND event IN ('game_start', 'round_change', 'heartbeat', 'game_end')
AND player_count >= 0 AND (event = 'game_start' OR round >= 0)
ORDER BY id DESC LIMIT ?`).all(server.name, cutoff, server.created_at, roundSince, matchSince,
ORDER BY id DESC LIMIT ?`
)
.all(
server.name,
cutoff,
server.created_at,
roundSince,
matchSince,
new Date(Date.parse(until) + 120_000).toISOString(),
(telemetryRetention().maxPerServer || 10000)) as ChatRoundReport[];
telemetryRetention().maxPerServer || 10000
) as ChatRoundReport[];
// Saving gameplay does not depend on diagnostic logging succeeding. Keep the
// surviving accepted snapshot useful when its request log was not recorded.
const latest = handle.prepare(`SELECT received_at AS receivedAt, payload FROM server_telemetry
WHERE server = ? AND received_at >= ? AND julianday(received_at) >= julianday(?)`)
.get(server.name, cutoff, server.created_at) as { receivedAt: string; payload: string | null } | undefined;
const latest = handle
.prepare(
`SELECT received_at AS receivedAt, payload FROM server_telemetry
WHERE server = ? AND received_at >= ? AND julianday(received_at) >= julianday(?)`
)
.get(server.name, cutoff, server.created_at) as
{ receivedAt: string; payload: string | null } | undefined;
if (latest?.payload) {
try {
const event = parseGameEvent(JSON.parse(latest.payload));
const round = event.event === 'game_start' ? null : event.round;
if (event.server === server.name && !reports.some(report => report.receivedAt === latest.receivedAt
&& report.event === event.event && report.round === round && report.playerCount === event.players.length)) {
reports.push({ receivedAt: latest.receivedAt, event: event.event, round, playerCount: event.players.length });
if (
event.server === server.name &&
!reports.some(
(report) =>
report.receivedAt === latest.receivedAt &&
report.event === event.event &&
report.round === round &&
report.playerCount === event.players.length
)
) {
reports.push({
receivedAt: latest.receivedAt,
event: event.event,
round,
playerCount: event.players.length
});
}
} catch { /* Legacy malformed snapshots cannot establish chat context. */ }
} catch {
/* Legacy malformed snapshots cannot establish chat context. */
}
}
return enrichGameChatTimeline(tail.lines, {
mtime: tail.mtime,
matches: matches.map(match => ({ ...match, startObserved: Boolean(match.startObserved) })),
matches: matches.map((match) => ({
...match,
startObserved: Boolean(match.startObserved)
})),
reports
});
}
export function matchChatTimeline(matchId: number): ContextualChatEntry[] {
return (db().prepare('SELECT entry FROM match_chat WHERE match_id = ? ORDER BY rowid')
.all(matchId) as { entry: string }[]).map(row => JSON.parse(row.entry));
return (
db()
.prepare('SELECT entry FROM match_chat WHERE match_id = ? ORDER BY rowid')
.all(matchId) as { entry: string }[]
).map((row) => JSON.parse(row.entry));
}
/** Save before filtering the live view so completed matches retain their messages. */
@@ -86,15 +148,24 @@ function archiveChat(entries: ContextualChatEntry[]): void {
})();
}
export function currentMatchChat(server: ServerRow, tail: TailResult | null): ContextualChatEntry[] {
export function currentMatchChat(
server: ServerRow,
tail: TailResult | null
): ContextualChatEntry[] {
const entries = tail ? serverChatTimeline(server, tail) : [];
archiveChat(entries);
return currentChatEntries(server, entries);
}
function currentChatEntries(server: ServerRow, entries: ContextualChatEntry[]): ContextualChatEntry[] {
const match = db().prepare(`SELECT id, started_at AS startedAt FROM leaderboard_matches
WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1`)
function currentChatEntries(
server: ServerRow,
entries: ContextualChatEntry[]
): ContextualChatEntry[] {
const match = db()
.prepare(
`SELECT id, started_at AS startedAt FROM leaderboard_matches
WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1`
)
.get(server.name) as { id: number; startedAt: string } | undefined;
let startIndex = entries.length - 1;
while (startIndex >= 0) {
@@ -105,27 +176,44 @@ function currentChatEntries(server: ServerRow, entries: ContextualChatEntry[]):
const current = entries.slice(Math.max(0, startIndex));
if (!match) {
// Servers without gameplay telemetry still have log-defined sessions.
return current.filter(entry => entry.context.realTime !== null
&& Date.parse(entry.context.realTime) >= Date.parse(server.created_at));
return current.filter(
(entry) =>
entry.context.realTime !== null &&
Date.parse(entry.context.realTime) >= Date.parse(server.created_at)
);
}
const start = entries[startIndex];
// A log restart can arrive before game_start telemetry. Clear immediately
// instead of falling back to messages from the previous recorded match.
if (start && !current.some(entry => entry.matchId === match.id)
&& (!start.context.realTime || Date.parse(start.context.realTime) > Date.parse(match.startedAt) + 1000)) return [];
if (
start &&
!current.some((entry) => entry.matchId === match.id) &&
(!start.context.realTime ||
Date.parse(start.context.realTime) > Date.parse(match.startedAt) + 1000)
)
return [];
return matchChatTimeline(match.id);
}
const pending = new Map<string, Promise<{ entries: ContextualChatEntry[]; missing: boolean }>>();
interface ChatReadState {
key: string; cursor?: LogCursor; lines: string[]; partial: Buffer; skipLine: boolean;
signature?: string; entries: ContextualChatEntry[]; contextVersion?: string; wasOccupied: boolean;
key: string;
cursor?: LogCursor;
lines: string[];
partial: Buffer;
skipLine: boolean;
signature?: string;
entries: ContextualChatEntry[];
contextVersion?: string;
wasOccupied: boolean;
}
const readers = new Map<string, ChatReadState>();
const failures = new Map<string, number>();
/** Shared by the live API and the worker; collection does not require a viewer. */
export function collectServerChat(server: ServerRow): Promise<{ entries: ContextualChatEntry[]; missing: boolean }> {
export function collectServerChat(
server: ServerRow
): Promise<{ entries: ContextualChatEntry[]; missing: boolean }> {
const key = `${server.name}:${server.created_at}:${server.native_instance_id}:${server.instance_generation}`;
const existing = pending.get(key);
if (existing) return existing;
@@ -135,12 +223,26 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
let saved = readers.get(server.name);
if (saved?.key !== key) saved = undefined;
// Publish the new cursor only after its messages have been processed.
const state: ChatReadState = saved ? { ...saved } : { key, lines: [], partial: Buffer.alloc(0), skipLine: false, entries: [], wasOccupied: false };
const state: ChatReadState = saved
? { ...saved }
: {
key,
lines: [],
partial: Buffer.alloc(0),
skipLine: false,
entries: [],
wasOccupied: false
};
const tail = await readLogChunk(path, state.cursor);
const current = getServer(server.name);
// A remote read may finish after destruction or replacement.
if (!current || current.destroyed_at || current.created_at !== server.created_at
|| current.native_instance_id !== server.native_instance_id || current.instance_generation !== server.instance_generation) {
if (
!current ||
current.destroyed_at ||
current.created_at !== server.created_at ||
current.native_instance_id !== server.native_instance_id ||
current.instance_generation !== server.instance_generation
) {
return { entries: [], missing: true };
}
if (!tail) {
@@ -149,22 +251,40 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
}
let changed = false;
if ('cursor' in tail) {
if (tail.reset) { state.lines = []; state.partial = Buffer.alloc(0); state.skipLine = tail.truncated; state.entries = []; }
if (tail.reset) {
state.lines = [];
state.partial = Buffer.alloc(0);
state.skipLine = tail.truncated;
state.entries = [];
}
state.cursor = tail.cursor;
const bytes = Buffer.concat([state.partial, Buffer.from(tail.contentBase64, 'base64')]);
const end = bytes.lastIndexOf(10) + 1;
if (end) {
const lines = bytes.subarray(0, end).toString('utf8').split('\n').slice(0, -1).map(line => line.replace(/\r$/, ''));
const lines = bytes
.subarray(0, end)
.toString('utf8')
.split('\n')
.slice(0, -1)
.map((line) => line.replace(/\r$/, ''));
if (state.skipLine) lines.shift();
state.skipLine = false;
state.lines = [...state.lines, ...lines].slice(-2000);
let length = 0, start = state.lines.length;
while (start > 0 && (length += Buffer.byteLength(state.lines[start - 1]) + 1) <= 256 * 1024) start--;
let length = 0,
start = state.lines.length;
while (
start > 0 &&
(length += Buffer.byteLength(state.lines[start - 1]) + 1) <= 256 * 1024
)
start--;
state.lines = state.lines.slice(start);
changed = true;
}
state.partial = Buffer.from(bytes.subarray(end));
if (state.partial.length > 256 * 1024) { state.partial = Buffer.alloc(0); state.skipLine = true; }
if (state.partial.length > 256 * 1024) {
state.partial = Buffer.alloc(0);
state.skipLine = true;
}
// File mtime anchors the final line only after the backlog has been read.
if (tail.cursor.offset < tail.size || state.partial.length) {
state.signature = undefined;
@@ -177,9 +297,21 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
if (changed) state.lines = tail.lines;
}
const signature = `${tail.size}:${tail.mtime}`;
const needsContext = state.entries.some(entry => entry.type === 'message' && entry.matchId === undefined);
const contextVersion = needsContext ? (db().prepare('SELECT received_at FROM server_telemetry WHERE server=?').get(server.name) as { received_at: string } | undefined)?.received_at : undefined;
if (changed || state.signature === undefined || (needsContext && contextVersion !== state.contextVersion)) {
const needsContext = state.entries.some(
(entry) => entry.type === 'message' && entry.matchId === undefined
);
const contextVersion = needsContext
? (
db()
.prepare('SELECT received_at FROM server_telemetry WHERE server=?')
.get(server.name) as { received_at: string } | undefined
)?.received_at
: undefined;
if (
changed ||
state.signature === undefined ||
(needsContext && contextVersion !== state.contextVersion)
) {
state.entries = serverChatTimeline(current, { ...tail, lines: state.lines });
archiveChat(state.entries);
state.contextVersion = contextVersion;
@@ -193,15 +325,16 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
}
export async function collectFleetChat(): Promise<void> {
const hosts = new Map(listHosts().map(host => [host.id, host]));
const hosts = new Map(listHosts().map((host) => [host.id, host]));
const observe = observationReader();
const live = listServers().filter(server => !server.destroyed_at);
const names = new Set(live.map(server => server.name));
const live = listServers().filter((server) => !server.destroyed_at);
const names = new Set(live.map((server) => server.name));
for (const name of readers.keys()) if (!names.has(name)) readers.delete(name);
for (const name of failures.keys()) if (!names.has(name)) failures.delete(name);
const servers = live.filter(server => {
const servers = live.filter((server) => {
if (server.host_id && !hosts.get(server.host_id)?.connected) return false;
if (server.container_state && !['running', 'unknown'].includes(server.container_state)) return false;
if (server.container_state && !['running', 'unknown'].includes(server.container_state))
return false;
const players = observe(server).last_players ?? server.last_players;
// One final read after players leave retains messages written at match end.
return players !== 0 || readers.get(server.name)?.wasOccupied;
@@ -212,7 +345,9 @@ export async function collectFleetChat(): Promise<void> {
try {
await collectServerChat(server);
const state = readers.get(server.name);
if (state) state.wasOccupied = (observe(server).last_players ?? server.last_players ?? 0) > 0;
if (state)
state.wasOccupied =
(observe(server).last_players ?? server.last_players ?? 0) > 0;
failures.delete(server.name);
} catch (error) {
if (Date.now() - (failures.get(server.name) ?? -Infinity) >= 5 * 60_000) {
src/lib/server/fleet/collections.ts
+420 -129
View File
@@ -1,7 +1,12 @@
import { scalingOwnershipBlocker } from '../autoscaling/ownership.js';
import { fleetSettings } from '../settings.js';
import { randomUUID } from 'node:crypto';
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
import type {
CollectionDetail,
CollectionMode,
CollectionPreview,
CollectionSummary
} from '$lib/deployment-plan.js';
import { db } from '../db/index.js';
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
import { creationQueue, destroyServer } from '../provisioning/index.js';
@@ -18,97 +23,203 @@ import { canCancelCreation, cancelCreation } from '../provisioning/cancel-creati
import { canCancelRemoval, cancelRemoval } from '../provisioning/history.js';
function coverage(entries: BlueprintEntry[], includeHealth = true) {
type Server = { name: string; game: string | null; map_slug: string | null; map: string | null };
const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[];
const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
type Server = {
name: string;
game: string | null;
map_slug: string | null;
map: string | null;
};
const registered = db()
.prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL')
.all() as Server[];
const queued = db()
.prepare(
`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[];
const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server));
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`
)
.all() as Server[];
const removing = new Set(
(
db()
.prepare(
"SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')"
)
.all() as { server: string }[]
).map((row) => row.server)
);
// A provisioning server can appear in both tables; count its identity once.
const servers = [...new Map([...queued, ...registered].map(server => [server.name, server])).values()];
return entries.map(entry => {
const existingServers = servers.filter(server => {
if (removing.has(server.name)) return false;
const parsed = parseServerName(server.name);
const game = server.game ?? parsed?.game;
if (game !== entry.recipe.game) return false;
if (server.map_slug) return server.map_slug === entry.recipe.mapSlug;
const namedMap = parsed?.mapCanonical ? zombieMapBySlug(game, parsed.mapSlug) : undefined;
if (namedMap && (!server.map || server.map === namedMap.canonical)) return namedMap.slug === entry.recipe.mapSlug;
// Legacy canonical maps are usable only when they identify one variant.
const matches = zombieMaps(game).filter(map => map.canonical === server.map);
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
}).map(server => server.name).sort();
const coverage = includeHealth ? existingServers.map(name => {
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
: server ? serverObservation(server).health : 'offline',
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
}) : [];
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
const servers = [
...new Map([...queued, ...registered].map((server) => [server.name, server])).values()
];
return entries.map((entry) => {
const existingServers = servers
.filter((server) => {
if (removing.has(server.name)) return false;
const parsed = parseServerName(server.name);
const game = server.game ?? parsed?.game;
if (game !== entry.recipe.game) return false;
if (server.map_slug) return server.map_slug === entry.recipe.mapSlug;
const namedMap = parsed?.mapCanonical
? zombieMapBySlug(game, parsed.mapSlug)
: undefined;
if (namedMap && (!server.map || server.map === namedMap.canonical))
return namedMap.slug === entry.recipe.mapSlug;
// Legacy canonical maps are usable only when they identify one variant.
const matches = zombieMaps(game).filter((map) => map.canonical === server.map);
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
})
.map((server) => server.name)
.sort();
const coverage = includeHealth
? existingServers.map((name) => {
const operation = db()
.prepare(
"SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1"
)
.get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return {
name,
health: operation
? operation.state === 'queued'
? 'queued'
: 'starting'
: server?.container_state &&
['stopped', 'exited', 'created', 'dead'].includes(
server.container_state
)
? 'stopped'
: server
? serverObservation(server).health
: 'offline',
href: operation
? `/provisioning/${encodeURIComponent(operation.id)}`
: `/servers/${encodeURIComponent(name)}`
};
})
: [];
return {
existingServers,
coverage,
existing: existingServers.length,
missing: Math.max(0, entry.count - existingServers.length)
};
});
}
function fleetSnapshot() {
return housePartyTargets().map(name => ({ name,
server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
return housePartyTargets().map((name) => ({
name,
server:
db()
.prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?')
.get(name) ?? null,
creation:
db()
.prepare(
"SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1"
)
.get(name) ?? null,
removal:
db()
.prepare(
"SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')"
)
.get(name) ?? null
}));
}
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, includeHealth = true) {
const counts = coverage(entries, includeHealth);
// Preserve deployed lobbies before queued ones, then the lowest lobby IDs.
const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers].sort((a, b) =>
Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true })
).slice(0, mode === 'exact' ? entries[index].count : undefined));
const retainedServers = counts.flatMap((entry, index) =>
[...entry.existingServers]
.sort(
(a, b) =>
Number(!!getServer(b)) - Number(!!getServer(a)) ||
a.localeCompare(b, 'en', { numeric: true })
)
.slice(0, mode === 'exact' ? entries[index].count : undefined)
);
const retained = new Set(retainedServers);
const matching = new Set(counts.flatMap(entry => entry.existingServers));
const removals = mode === 'exact' ? housePartyTargets().filter(name => !retained.has(name)).map(name => ({ name,
reason: matching.has(name) ? 'Exceeds the target lobby count' : 'Outside this collection or already awaiting removal'
})) : [];
const matching = new Set(counts.flatMap((entry) => entry.existingServers));
const removals =
mode === 'exact'
? housePartyTargets()
.filter((name) => !retained.has(name))
.map((name) => ({
name,
reason: matching.has(name)
? 'Exceeds the target lobby count'
: 'Outside this collection or already awaiting removal'
}))
: [];
return { counts, retainedServers, removals };
}
function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] {
const occupied = db().prepare(`SELECT name FROM servers UNION
function allocate(
entries: BlueprintEntry[],
counts = coverage(entries)
): CollectionPreview['servers'] {
const occupied = db()
.prepare(
`SELECT name FROM servers UNION
SELECT server AS name FROM native_instances WHERE retired_at IS NULL UNION
SELECT server_id AS name FROM server_key_leases UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`)
OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`
)
.all() as { name: string }[];
const lobbies = new Map<string, Set<number>>();
for (const { name } of occupied) {
const match = /^(.*)_(\d+)$/.exec(name);
if (!match) continue;
const taken = lobbies.get(match[1]) ?? new Set<number>();
taken.add(Number(match[2])); lobbies.set(match[1], taken);
taken.add(Number(match[2]));
lobbies.set(match[1], taken);
}
const availableKeys = { t5: availableServerKeyCount('t5'), t6: availableServerKeyCount('t6') };
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const taken = lobbies.get(base) ?? new Set<number>();
const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null;
const name = lobby === null ? null : `${base}_${lobby}`;
const native = entry.recipe.native;
const port = null;
if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0;
if (hasSavedKey) availableKeys[entry.recipe.game]--;
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey };
}));
return entries.flatMap((entry, entryIndex) =>
Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const taken = lobbies.get(base) ?? new Set<number>();
const lobby =
Array.from({ length: 20 }, (_, index) => index + 1).find(
(number) => !taken.has(number)
) ?? null;
const name = lobby === null ? null : `${base}_${lobby}`;
const native = entry.recipe.native;
const port = null;
if (lobby !== null) {
taken.add(lobby);
lobbies.set(base, taken);
}
const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0;
if (hasSavedKey) availableKeys[entry.recipe.game]--;
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey };
})
);
}
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
function blockersFor(
servers: CollectionPreview['servers'],
entries: BlueprintEntry[] = []
): string[] {
const blockers: string[] = [];
for (const server of servers) if (server.name) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); }
for (const server of servers)
if (server.name) {
const owned = scalingOwnershipBlocker(server.name);
if (owned) blockers.push(owned);
}
if (!servers.length) return blockers;
if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.');
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
if (servers.some((server) => !server.native && !entries[server.entryIndex]?.recipe.native))
blockers.push('Select and prepare a host before launching.');
if (servers.some((server) => !server.name))
blockers.push(
'There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.'
);
return blockers;
}
@@ -116,11 +227,16 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
const blockers: string[] = [];
const native = (name: string) => {
if (nativeInstance(name)) return true;
const creation = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined;
const creation = db()
.prepare(
`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`
)
.get(name) as { id: string } | undefined;
return !!creation && nativeDeployment(creation.id);
};
if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.');
if (removals.some((server) => !native(server.name)))
blockers.push('Every server to remove must have a native host instance.');
for (const server of removals) {
const owned = scalingOwnershipBlocker(server.name);
if (owned) blockers.push(owned);
@@ -130,50 +246,118 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
return blockers;
}
export function previewCollection(blueprintId: unknown, mode: unknown = 'fill', selection?: Record<string, unknown>): CollectionPreview {
if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.');
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
export function previewCollection(
blueprintId: unknown,
mode: unknown = 'fill',
selection?: Record<string, unknown>
): CollectionPreview {
if (typeof blueprintId !== 'string' || !blueprintId)
throw new Error('Choose a collection to launch.');
if (mode !== 'fill' && mode !== 'exact')
throw new Error('Choose fill or exact collection mode.');
return db().transaction((): CollectionPreview => {
const { entries, blueprintName } = blueprintEntries(blueprintId);
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
for (const entry of entries) {
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
entry.recipe.native = nativeLaunchSelection(
{ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId },
entry.recipe.game
);
entry.recipe.image = entry.recipe.native.image!;
}
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
const servers = allocate(entries, counts);
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
const selections = servers.flatMap(server => server.native ? [server.native] : []);
const selections = servers.flatMap((server) => (server.native ? [server.native] : []));
const availability = selections.length ? launchAvailability(selections) : {};
const id = randomUUID();
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null);
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ...availability, ready: blockers.length === 0,
entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) };
db()
.prepare(
'DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL'
)
.run(new Date().toISOString());
db()
.prepare(
'INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)'
)
.run(
id,
expiresAt,
blueprintName,
JSON.stringify(entries),
JSON.stringify(servers),
mode,
mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null
);
return {
id,
expiresAt,
blueprintName,
mode,
retainedServers,
removals,
launchConcurrency: fleetSettings().createConcurrency,
servers,
blockers,
...availability,
ready: blockers.length === 0,
entries: entries.map((entry, index) => ({
count: entry.count,
...counts[index],
recipe: summarizeRecipe(entry.recipe)
}))
};
})();
}
/** Save all reviewed identities and keys atomically; native placement belongs to provisioning. */
export function queueCollection(input: { previewId?: unknown; keys?: unknown }): { ok: true; collectionId: string | null } {
if (typeof input.previewId !== 'string' || !input.previewId) throw new Error('Review a collection preview before launching.');
if (input.keys !== undefined && (!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))) throw new Error('Provide keys indexed by the exact server ID.');
export function queueCollection(input: { previewId?: unknown; keys?: unknown }): {
ok: true;
collectionId: string | null;
} {
if (typeof input.previewId !== 'string' || !input.previewId)
throw new Error('Review a collection preview before launching.');
if (
input.keys !== undefined &&
(!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))
)
throw new Error('Provide keys indexed by the exact server ID.');
const keys = (input.keys ?? {}) as Record<string, unknown>;
return db().transaction(() => {
const preview = db().prepare('SELECT * FROM server_collection_previews WHERE id = ?').get(input.previewId) as {
id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null; mode: CollectionMode; fleet_snapshot: string | null;
} | undefined;
if (!preview) throw new Error('Collection preview expired or was not found. Refresh the preview.');
if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id };
if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.');
const preview = db()
.prepare('SELECT * FROM server_collection_previews WHERE id = ?')
.get(input.previewId) as
| {
id: string;
expires_at: string;
blueprint_name: string;
entries: string;
servers: string;
collection_id: string | null;
mode: CollectionMode;
fleet_snapshot: string | null;
}
| undefined;
if (!preview)
throw new Error('Collection preview expired or was not found. Refresh the preview.');
if (preview.collection_id)
return { ok: true as const, collectionId: preview.collection_id };
if (preview.expires_at <= new Date().toISOString())
throw new Error('Collection preview expired. Refresh it before launching.');
const entries: BlueprintEntry[] = JSON.parse(preview.entries);
// Older previews embedded plugin bytes. Queued jobs need only the
// reviewed metadata; provisioning captures the current files at dispatch.
for (const entry of entries) entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file);
for (const entry of entries)
entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file);
const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers);
if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) {
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
if (
preview.mode === 'exact' &&
preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())
) {
throw new Error(
'The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.'
);
}
const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false);
const servers = reviewed;
@@ -181,7 +365,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
for (const server of servers) {
const recipe = entries[server.entryIndex].recipe;
const selection = server.native ?? recipe.native;
if (!selection) throw new Error('Select a native host before launching. Refresh the preview.');
if (!selection)
throw new Error('Select a native host before launching. Refresh the preview.');
const identity = JSON.stringify([recipe.game, selection]);
if (validatedSelections.has(identity)) continue;
// Recheck saved configuration once per selection without allocating
@@ -191,50 +376,102 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
}
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
if (blockers.length) throw new Error(blockers.join(' '));
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) {
throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.');
if (
entries.some(
(_entry, index) =>
counts[index].missing !==
reviewed.filter((server) => server.entryIndex === index).length
)
) {
throw new Error(
'Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.'
);
}
const expected = new Set(servers.map(server => server.name));
const expected = new Set(servers.map((server) => server.name));
for (const [name, value] of Object.entries(keys)) {
if (!expected.has(name) || typeof value !== 'string') throw new Error('Keys must be strings for server IDs shown in this preview.');
if (!expected.has(name) || typeof value !== 'string')
throw new Error('Keys must be strings for server IDs shown in this preview.');
}
if (!servers.length && !removals.length) return { ok: true as const, collectionId: null };
const collectionId = randomUUID();
db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)')
.run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers));
db()
.prepare(
'INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)'
)
.run(
collectionId,
preview.blueprint_name,
new Date().toISOString(),
preview.mode,
JSON.stringify(retainedServers)
);
removals.forEach((server, position) => {
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
const queued = db()
.prepare(
"SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'"
)
.get(server.name) as { id: string } | undefined;
let id: string;
if (queued && !getServer(server.name)) {
// No container or files exist yet: cancel the reservation instead of booting just to remove it.
const result = destroyServer(server.name, 'blueprint-exact');
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
if (!result.deploymentId)
throw new Error('Could not queue removal for ' + server.name);
id = result.deploymentId;
db().prepare("UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?").run(new Date().toISOString(), queued.id);
recordDeploymentEvent(queued.id, 'Queued creation cancelled by exact collection; saved key retained.');
db()
.prepare(
"UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?"
)
.run(new Date().toISOString(), queued.id);
recordDeploymentEvent(
queued.id,
'Queued creation cancelled by exact collection; saved key retained.'
);
deploymentStage(id, 'Cancelled queued creation; no server was started');
finishDeployment(id, { ok: true });
} else {
const result = destroyServer(server.name, 'blueprint-exact');
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
if (!result.deploymentId)
throw new Error('Could not queue removal for ' + server.name);
id = result.deploymentId;
}
db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);
db()
.prepare(
'INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)'
)
.run(collectionId, id, position);
});
servers.forEach((server, index) => {
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.');
const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name });
const recipe = {
...entries[server.entryIndex].recipe,
...(server.native ? { native: server.native } : {})
};
if (!recipe.native)
throw new Error('Select a native host before launching. Refresh the preview.');
const result = queueNativeCreationIntent({
server: server.name!,
game: recipe.game,
mapSlug: recipe.mapSlug,
selection: recipe.native,
recipe,
plutoKey: keys[server.name!] as string | undefined,
createdBy: 'ui',
blueprintName: preview.blueprint_name
});
// Throw so the outer transaction also reverses all earlier keys and children.
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
if (!result.ok)
throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db()
.prepare(
'INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)'
)
.run(collectionId, result.deploymentId, index);
});
db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id);
db()
.prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?')
.run(collectionId, preview.id);
return { ok: true as const, collectionId };
})();
}
@@ -243,39 +480,60 @@ const membershipSql = `SELECT collection_id, deployment_id, position FROM server
UNION ALL SELECT collection_id, deployment_id, position FROM server_collection_removals`;
function collectionCancellationCandidates(id: string) {
const creations = (db().prepare(`SELECT d.id FROM server_collection_members m
const creations = (
db()
.prepare(
`SELECT d.id FROM server_collection_members m
JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.collection_id=?
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.creation_id=d.id)`)
.all(id) as { id: string }[]).filter(job => canCancelCreation(job.id));
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.creation_id=d.id)`
)
.all(id) as { id: string }[]
).filter((job) => canCancelCreation(job.id));
// Exact launches can reuse manual removals or creation cleanup jobs. Membership
// alone must never give a launch permission to withdraw those operations.
const removals = (db().prepare(`SELECT d.id FROM server_collection_removals m
const removals = (
db()
.prepare(
`SELECT d.id FROM server_collection_removals m
JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.collection_id=? AND d.triggered_by='blueprint-exact'
AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.operation_id IS NOT NULL)
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.cleanup_id=d.id)`)
.all(id) as { id: string }[]).filter(job => canCancelRemoval(job.id));
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.cleanup_id=d.id)`
)
.all(id) as { id: string }[]
).filter((job) => canCancelRemoval(job.id));
return { creations, removals };
}
/** Withdraw the remaining launch atomically; partial instances keep durable cleanup jobs. */
export function cancelCollection(id: string): boolean {
return db().transaction(() => {
const collection = db().prepare('SELECT cancelled_at FROM server_launch_collections WHERE id=?').get(id) as { cancelled_at: string | null } | undefined;
const collection = db()
.prepare('SELECT cancelled_at FROM server_launch_collections WHERE id=?')
.get(id) as { cancelled_at: string | null } | undefined;
if (!collection) return false;
if (collection.cancelled_at) return true;
const { creations, removals } = collectionCancellationCandidates(id);
if (!creations.length && !removals.length) return false;
db().prepare('UPDATE server_launch_collections SET cancelled_at=? WHERE id=?').run(new Date().toISOString(), id);
db()
.prepare('UPDATE server_launch_collections SET cancelled_at=? WHERE id=?')
.run(new Date().toISOString(), id);
for (const creation of creations) {
if (!cancelCreation(creation.id, 'collection-cancel')) throw new Error('Could not cancel a collection creation. Refresh the launch and try again.');
if (!cancelCreation(creation.id, 'collection-cancel'))
throw new Error(
'Could not cancel a collection creation. Refresh the launch and try again.'
);
}
for (const removal of removals) {
// A shared removal remains requested until every owning launch withdraws.
const shared = db().prepare(`SELECT 1 FROM server_collection_removals m
const shared = db()
.prepare(
`SELECT 1 FROM server_collection_removals m
JOIN server_launch_collections c ON c.id=m.collection_id
WHERE m.deployment_id=? AND c.id!=? AND c.cancelled_at IS NULL LIMIT 1`).get(removal.id, id);
WHERE m.deployment_id=? AND c.id!=? AND c.cancelled_at IS NULL LIMIT 1`
)
.get(removal.id, id);
if (!shared) cancelRemoval(removal.id);
}
return true;
@@ -299,26 +557,59 @@ const summarySql = `SELECT c.id, c.name, c.created_at, c.cancelled_at, c.mode, C
LEFT JOIN server_deployments cleanup ON cleanup.id = cc.cleanup_id`;
export function listCollections(): CollectionSummary[] {
return db().prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC').all() as CollectionSummary[];
return db()
.prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC')
.all() as CollectionSummary[];
}
export function getCollection(id: string): CollectionDetail | null {
const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined;
const collection = db()
.prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id')
.get(id) as CollectionSummary | undefined;
if (!collection) return null;
const servers = db().prepare(`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
const servers = db()
.prepare(
`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
(d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers'];
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`
)
.all(id) as CollectionDetail['servers'];
const queue = creationQueue();
const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1]));
const stored = db().prepare('SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string; retained_server_deletions: string };
const cancellations = db().prepare(`SELECT cc.creation_id, cc.cleanup_id AS cleanupId, d.state, d.stage, d.error
const stored = db()
.prepare(
'SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?'
)
.get(id) as { retained_servers: string; retained_server_deletions: string };
const cancellations = db()
.prepare(
`SELECT cc.creation_id, cc.cleanup_id AS cleanupId, d.state, d.stage, d.error
FROM server_collection_members m JOIN native_creation_cancellations cc ON cc.creation_id=m.deployment_id
JOIN server_deployments d ON d.id=cc.cleanup_id WHERE m.collection_id=?`).all(id) as
(NonNullable<CollectionDetail['servers'][number]['cancellation']> & { creation_id: string })[];
const cleanupByCreation = new Map(cancellations.map(({ creation_id, ...cleanup }) => [creation_id, cleanup]));
JOIN server_deployments d ON d.id=cc.cleanup_id WHERE m.collection_id=?`
)
.all(id) as (NonNullable<CollectionDetail['servers'][number]['cancellation']> & {
creation_id: string;
})[];
const cleanupByCreation = new Map(
cancellations.map(({ creation_id, ...cleanup }) => [creation_id, cleanup])
);
const candidates = collection.cancelled_at ? null : collectionCancellationCandidates(id);
return { collection, canCancel: !!candidates && (candidates.creations.length > 0 || candidates.removals.length > 0),
retainedServers: JSON.parse(stored.retained_servers), retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length },
servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null, cancellation: cleanupByCreation.get(server.id) ?? null })) };
return {
collection,
canCancel:
!!candidates && (candidates.creations.length > 0 || candidates.removals.length > 0),
retainedServers: JSON.parse(stored.retained_servers),
retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions),
queue: {
concurrency: fleetSettings().createConcurrency,
active: queue.active,
queued: queue.queued.length
},
servers: servers.map((server) => ({
...server,
queuePosition: positions.get(server.id) ?? null,
cancellation: cleanupByCreation.get(server.id) ?? null
}))
};
}
src/lib/server/fleet/config-inspect.ts
+72 -19
View File
@@ -1,62 +1,115 @@
import { composeServerCfg, parseCfg } from './config-compose.js';
import { selectedZombieMap, mapRotation } from './map-aliases.js';
import { configWarnings, type ConfigAssignment, type ConfigInspection, type ConfigSource } from '../../config-workbench.js';
import {
configWarnings,
type ConfigAssignment,
type ConfigInspection,
type ConfigSource
} from '../../config-workbench.js';
export interface ConfigLayer { source: ConfigSource; text: string; }
export interface ConfigLayer {
source: ConfigSource;
text: string;
}
/** Trace file composition, including assignments left in place by the composer.
* This is an intended-file preview, not a query of live runtime dvars. */
export function inspectConfig(baseline: ConfigLayer, overlays: ConfigLayer[], game: string | null, map: string | null, mapSlug?: string | null): ConfigInspection {
export function inspectConfig(
baseline: ConfigLayer,
overlays: ConfigLayer[],
game: string | null,
map: string | null,
mapSlug?: string | null
): ConfigInspection {
const sorted = [...overlays].sort((a, b) => a.source.id! - b.source.id!);
const parsed = parseCfg(baseline.text);
type TracedLine = { line: string; history: ConfigAssignment[] };
function assignment(line: string, source: ConfigSource, index: number): ConfigAssignment {
const match = /^\s*(?:(?:set|seta)\s+)(\S+)\s+(.*)$/i.exec(line)
?? /^\s*(sv_maprotation)\s+(.*)$/i.exec(line);
const match =
/^\s*(?:(?:set|seta)\s+)(\S+)\s+(.*)$/i.exec(line) ??
/^\s*(sv_maprotation)\s+(.*)$/i.exec(line);
return { value: match?.[2] ?? '', source, line: index + 1 };
}
const lines: TracedLine[] = parsed.order.map((entry, index) => ({
line: entry.line, history: [assignment(entry.line, baseline.source, index)]
line: entry.line,
history: [assignment(entry.line, baseline.source, index)]
}));
const replacements = new Map<string, TracedLine>();
const warnings = configWarnings(baseline.text, false).map(w => `${baseline.source.name}: ${w}`);
const warnings = configWarnings(baseline.text, false).map(
(w) => `${baseline.source.name}: ${w}`
);
for (const overlay of sorted) {
warnings.push(...configWarnings(overlay.text, true).map(w => `${overlay.source.name}: ${w}`));
warnings.push(
...configWarnings(overlay.text, true).map((w) => `${overlay.source.name}: ${w}`)
);
overlay.text.split(/\r?\n/).forEach((line, index) => {
const match = /^\s*set\s+(\S+)\s+/.exec(line);
if (!match) return;
const previous = replacements.get(match[1]);
replacements.set(match[1], { line, history: [...(previous?.history ?? []), assignment(line, overlay.source, index)] });
replacements.set(match[1], {
line,
history: [...(previous?.history ?? []), assignment(line, overlay.source, index)]
});
});
}
for (const [key, replacement] of replacements) {
const index = parsed.entries.get(key);
if (index === undefined) lines.push(replacement);
else lines[index] = { line: replacement.line, history: [...lines[index].history, ...replacement.history] };
else
lines[index] = {
line: replacement.line,
history: [...lines[index].history, ...replacement.history]
};
}
const sources = [baseline.source, ...sorted.map(o => o.source)];
const sources = [baseline.source, ...sorted.map((o) => o.source)];
const selected = selectedZombieMap(game, map, mapSlug);
if (selected) {
const source: ConfigSource = { id: null, name: 'Server map selection', version: null, kind: 'map' };
const source: ConfigSource = {
id: null,
name: 'Server map selection',
version: null,
kind: 'map'
};
sources.push(source);
const rotation = /^\s*(?:(?:set|seta)\s+)?sv_maprotation\s+/i;
const history = lines.filter(entry => rotation.test(entry.line)).flatMap(entry => entry.history);
for (let i = lines.length - 1; i >= 0; i--) if (rotation.test(lines[i].line)) lines.splice(i, 1);
const history = lines
.filter((entry) => rotation.test(entry.line))
.flatMap((entry) => entry.history);
for (let i = lines.length - 1; i >= 0; i--)
if (rotation.test(lines[i].line)) lines.splice(i, 1);
const line = `set sv_maprotation "${mapRotation(selected)}"`;
lines.push({ line, history: [...history, assignment(line, source, 0)] });
}
const byKey = new Map<string, { key: string; history: ConfigAssignment[] }>();
for (const entry of lines) {
const match = /^\s*(?:set|seta)\s+(\S+)\s+/i.exec(entry.line) ?? /^\s*(sv_maprotation)\s+/i.exec(entry.line);
const match =
/^\s*(?:set|seta)\s+(\S+)\s+/i.exec(entry.line) ??
/^\s*(sv_maprotation)\s+/i.exec(entry.line);
if (!match) continue;
const normalized = match[1].toLowerCase();
const existing = byKey.get(normalized);
byKey.set(normalized, { key: match[1], history: [...(existing?.history ?? []), ...entry.history] });
byKey.set(normalized, {
key: match[1],
history: [...(existing?.history ?? []), ...entry.history]
});
}
return {
text: composeServerCfg(baseline.text, sorted.map(o => ({ id: o.source.id!, text: o.text })), game, map, mapSlug),
sources, warnings,
settings: [...byKey.values()].map(({ key, history }) => ({ key, history, value: history.at(-1)!.value, source: history.at(-1)!.source }))
text: composeServerCfg(
baseline.text,
sorted.map((o) => ({ id: o.source.id!, text: o.text })),
game,
map,
mapSlug
),
sources,
warnings,
settings: [...byKey.values()]
.map(({ key, history }) => ({
key,
history,
value: history.at(-1)!.value,
source: history.at(-1)!.source
}))
.sort((a, b) => a.key.localeCompare(b.key))
};
}
src/lib/server/fleet/drift.ts
+360 -114
View File
@@ -1,6 +1,12 @@
import { putFileBlob } from '../db/file-blobs.js';
import { getVersionContent } from './scripts.js';
import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js';
import {
readdir,
readFile,
stat,
nativeFileManifest,
type FileFingerprint
} from '../backends/files.js';
import { randomUUID } from 'node:crypto';
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
import { join, posix, relative, sep } from 'node:path';
@@ -27,12 +33,18 @@ async function scanBatch<T, R>(items: readonly T[], work: (item: T) => Promise<R
async function pump() {
while (!failed && cursor < items.length) {
const index = cursor++;
try { results[index] = await work(items[index]); }
catch (error) { failed = true; throw error; }
try {
results[index] = await work(items[index]);
} catch (error) {
failed = true;
throw error;
}
}
}
const workers = await Promise.allSettled(Array.from({ length: Math.min(4, items.length) }, pump));
const failure = workers.find(result => result.status === 'rejected');
const workers = await Promise.allSettled(
Array.from({ length: Math.min(4, items.length) }, pump)
);
const failure = workers.find((result) => result.status === 'rejected');
if (failure?.status === 'rejected') throw failure.reason;
return results;
}
@@ -73,15 +85,22 @@ function toPosix(p: string): string {
}
function hasActiveBeta(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
return !!db()
.prepare(
`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`
)
.get(server);
}
async function requireServerDirectory(server: string): Promise<void> {
try {
if (!(await stat(serverDir(server))).isDirectory()) throw new Error('path is not a directory');
if (!(await stat(serverDir(server))).isDirectory())
throw new Error('path is not a directory');
} catch (error) {
throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`);
throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`, {
cause: error
});
}
}
@@ -92,7 +111,10 @@ async function readExistingFile(path: string): Promise<Buffer | null> {
});
}
async function fingerprint(path: string, manifest: Map<string, FileFingerprint> | null): Promise<FileFingerprint | null> {
async function fingerprint(
path: string,
manifest: Map<string, FileFingerprint> | null
): Promise<FileFingerprint | null> {
if (manifest) return manifest.get(path) ?? null;
const content = await readExistingFile(path);
return content === null ? null : { size: content.byteLength, sha256: sha256Buffer(content) };
@@ -102,10 +124,11 @@ async function walkFiles(root: string): Promise<string[]> {
const out: string[] = [];
let directories = [root];
while (directories.length) {
const children = await scanBatch(directories, async dir => {
const children = await scanBatch(directories, async (dir) => {
let entries;
try { entries = await readdir(dir, { withFileTypes: true }); }
catch (error) {
try {
entries = await readdir(dir, { withFileTypes: true });
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [];
throw error;
}
@@ -131,51 +154,83 @@ async function walkFiles(root: string): Promise<string[]> {
* artifacts with the promoted versions.
*/
export async function scanServerDrift(server: string): Promise<DriftEntry[]> {
return trackActivity(`drift:${server}`, `Checking files on ${server}`, () => scanServerFiles(server));
return trackActivity(`drift:${server}`, `Checking files on ${server}`, () =>
scanServerFiles(server)
);
}
async function scanServerFiles(server: string): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv || srv.destroyed_at || hasActiveBeta(server)) return [];
// A missing mount is unavailable evidence, not an empty managed tree.
const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) => join(destOfKind(server, kind), script.target_rel_path));
if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id) expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server)));
if (srv.created_by && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE));
const manifest = await nativeFileManifest(serverDir(server), DRIFT_DESTS.map(kind => destOfKind(server, kind)), [...DRIFT_IGNORE_FILENAMES], expectedPaths);
const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) =>
join(destOfKind(server, kind), script.target_rel_path)
);
if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id)
expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server)));
if (
srv.created_by &&
srv.game_mode === 'zombie' &&
getServerMapConfig(srv.game, srv.map, srv.map_slug)
)
expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE));
const manifest = await nativeFileManifest(
serverDir(server),
DRIFT_DESTS.map((kind) => destOfKind(server, kind)),
[...DRIFT_IGNORE_FILENAMES],
expectedPaths
);
if (!manifest) await requireServerDirectory(server);
const drift = await scanManagedForServer(server, manifest);
// Previous inventory is an audit of installed bytes, not permission to
// retain artifacts whose scope was moved or removed from the library.
const managed = new Set(promotedArtifactMetadata(srv.game, srv.map)
.map(({ kind, script }) => `${kind}:${script.target_rel_path}`));
const managed = new Set(
promotedArtifactMetadata(srv.game, srv.map).map(
({ kind, script }) => `${kind}:${script.target_rel_path}`
)
);
const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined;
if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`);
// Without a promoted baseline, Plutainer owns the seeded config. Keep it
// until a baseline is promoted; startup arguments handle map selection.
const keepDefaultConfig = srv?.game
&& selectedZombieMap(srv.game, srv.map, srv.map_slug)
&& !baseline?.current_version_id;
const keepDefaultConfig =
srv?.game &&
selectedZombieMap(srv.game, srv.map, srv.map_slug) &&
!baseline?.current_version_id;
// Scan for untracked files only after resolving managed targets.
for (const kind of DRIFT_DESTS) {
const dest = destOfKind(server, kind);
const onDisk = manifest
? [...manifest.keys()].filter(path => path.startsWith(`${dest}${sep}`)).map(path => toPosix(relative(dest, path)))
.filter(path => !path.split('/').some(name => DRIFT_IGNORE_FILENAMES.has(name))).sort()
? [...manifest.keys()]
.filter((path) => path.startsWith(`${dest}${sep}`))
.map((path) => toPosix(relative(dest, path)))
.filter(
(path) => !path.split('/').some((name) => DRIFT_IGNORE_FILENAMES.has(name))
)
.sort()
: await walkFiles(dest);
if (onDisk.length === 0) continue;
const untracked = await scanBatch(onDisk, async (relPath): Promise<DriftEntry | null> => {
if (managed.has(`${kind}:${relPath}`)) return null;
if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) return null;
if (kind === 'config' && relPath === MAP_CONFIG_FILE && srv?.created_by
&& srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) return null;
if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server))
return null;
if (
kind === 'config' &&
relPath === MAP_CONFIG_FILE &&
srv?.created_by &&
srv.game_mode === 'zombie' &&
getServerMapConfig(srv.game, srv.map, srv.map_slug)
)
return null;
const abs = join(dest, relPath);
try {
const info = await fingerprint(abs, manifest);
if (!info) return null;
return {
server, relPath,
server,
relPath,
...info,
kind,
driftKind: 'untracked'
@@ -196,38 +251,63 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
// Every promoted artifact whose scope matches this server's game/map should
// have its target file present on disk. Anything absent is 'missing' drift.
// Configs are handled as a composed layer (baseline + overlays → one cfg).
async function scanManagedForServer(server: string, manifest: Map<string, FileFingerprint> | null): Promise<DriftEntry[]> {
async function scanManagedForServer(
server: string,
manifest: Map<string, FileFingerprint> | null
): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv) return [];
const out: DriftEntry[] = [];
// Provisioned servers load this generated file after their main cfg. It
// remains managed even when there is no promoted baseline yet.
const mapConfig = srv.created_by && srv.game_mode === 'zombie'
? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null;
const mapConfig =
srv.created_by && srv.game_mode === 'zombie'
? getServerMapConfig(srv.game, srv.map, srv.map_slug)
: null;
if (mapConfig) {
const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE);
const content = await fingerprint(abs, manifest);
if (content?.sha256 !== sha256Buffer(mapConfig)) {
out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing',
size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) });
out.push({
server,
relPath: MAP_CONFIG_FILE,
kind: 'config',
driftKind: content ? 'replace' : 'missing',
size: mapConfig.byteLength,
sha256: sha256Buffer(mapConfig)
});
}
}
// Inventory records the last managed write, not the current disk bytes.
// Check both so edits and outdated versions are detected on tracked files.
const managed = await scanBatch(promotedArtifactMetadata(srv.game, srv.map), async ({ kind, script, version }): Promise<DriftEntry | null> => {
const content = await fingerprint(join(destOfKind(server, kind), script.target_rel_path), manifest);
const inventory = getServerScript(server, script.target_rel_path);
if (content?.sha256 === version.sha256 && inventory?.sha256 === version.sha256
&& inventory.script_version_id === version.id) return null;
return {
server, relPath: script.target_rel_path, kind,
driftKind: content ? 'replace' : 'missing',
size: version.size, sha256: version.sha256,
scriptId: script.id, scriptVersionId: version.id
};
});
const managed = await scanBatch(
promotedArtifactMetadata(srv.game, srv.map),
async ({ kind, script, version }): Promise<DriftEntry | null> => {
const content = await fingerprint(
join(destOfKind(server, kind), script.target_rel_path),
manifest
);
const inventory = getServerScript(server, script.target_rel_path);
if (
content?.sha256 === version.sha256 &&
inventory?.sha256 === version.sha256 &&
inventory.script_version_id === version.id
)
return null;
return {
server,
relPath: script.target_rel_path,
kind,
driftKind: content ? 'replace' : 'missing',
size: version.size,
sha256: version.sha256,
scriptId: script.id,
scriptVersionId: version.id
};
}
);
out.push(...managed.filter((entry): entry is DriftEntry => entry !== null));
// Composed cfg: the effective dedicated_sp.cfg is baseline + all promoted
@@ -247,13 +327,24 @@ async function scanManagedForServer(server: string, manifest: Map<string, FileFi
// Older launches tracked the promoted bytes before Plutainer
// applied the instance password. Reconcile only that known
// before-image, and only after verifying the full effective file.
if (composed.sha256 !== composed.promotedSha256 && content?.sha256 === composed.sha256
&& inv?.sha256 === composed.promotedSha256 && !hasActiveBeta(server)) {
recordScript(server, targetRel, inv.script_version_id, composed.sha256,
composed.content.byteLength, 'runtime-config-reconciliation');
if (
composed.sha256 !== composed.promotedSha256 &&
content?.sha256 === composed.sha256 &&
inv?.sha256 === composed.promotedSha256 &&
!hasActiveBeta(server)
) {
recordScript(
server,
targetRel,
inv.script_version_id,
composed.sha256,
composed.content.byteLength,
'runtime-config-reconciliation'
);
inv = getServerScript(server, targetRel);
}
const onDiskMatches = content?.sha256 === composed.sha256 && inv?.sha256 === composed.sha256;
const onDiskMatches =
content?.sha256 === composed.sha256 && inv?.sha256 === composed.sha256;
if (!onDiskMatches) {
out.push({
server,
@@ -282,7 +373,8 @@ async function resolveManagedEntry(server: string, entry: DriftEntry): Promise<B
if (entry.kind === 'config') {
const srv = getServer(server);
if (!srv?.game) return null;
if (entry.relPath === MAP_CONFIG_FILE) return getServerMapConfig(srv.game, srv.map, srv.map_slug);
if (entry.relPath === MAP_CONFIG_FILE)
return getServerMapConfig(srv.game, srv.map, srv.map_slug);
const composed = composeExpectedCfg(server, srv.game, srv.map, srv.map_slug);
return composed?.content ?? null;
}
@@ -299,7 +391,9 @@ function composeExpectedCfg(
): { content: Buffer; sha256: string; promotedSha256: string } | null {
const promoted = getPromotedServerConfig(game, map, mapSlug);
if (!promoted) return null;
const content = Buffer.from(applyRuntimePassword(promoted.toString('utf8'), runtimePasswordForServer(server)));
const content = Buffer.from(
applyRuntimePassword(promoted.toString('utf8'), runtimePasswordForServer(server))
);
return { content, sha256: sha256Buffer(content), promotedSha256: sha256Buffer(promoted) };
}
@@ -318,46 +412,70 @@ export async function scanFleetDrift(): Promise<DriftEntry[]> {
}
export interface RunDriftInput {
servers?: string[]; // if omitted, scans the whole fleet
servers?: string[]; // if omitted, scans the whole fleet
dryRun?: boolean;
triggeredBy?: 'ui' | 'sweep' | 'cron';
expectedFingerprint?: string;
}
export class DriftPlanChanged extends Error {
constructor(public plan: DriftPlan) { super('The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'); }
constructor(public plan: DriftPlan) {
super(
'The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'
);
}
}
function driftTargets(servers?: string[]): string[] {
const registered = listServers().map(server => server.name);
if (servers && (!servers.length || servers.some(name => !registered.includes(name)))) throw new Error('Select at least one registered server. Refresh the fleet if a server was removed.');
const registered = listServers().map((server) => server.name);
if (servers && (!servers.length || servers.some((name) => !registered.includes(name))))
throw new Error(
'Select at least one registered server. Refresh the fleet if a server was removed.'
);
return [...new Set(servers ?? registered)].sort();
}
async function describePlan(servers: string[], entries: DriftEntry[]): Promise<DriftPlan> {
const files: DriftPlanFile[] = [];
for (const entry of entries) {
const before = await readFile(join(destOfKind(entry.server, entry.kind), entry.relPath)).catch((error: NodeJS.ErrnoException) => {
const before = await readFile(
join(destOfKind(entry.server, entry.kind), entry.relPath)
).catch((error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return null;
throw error;
});
const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry);
files.push({ server: entry.server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null });
const after =
entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry);
files.push({
server: entry.server,
relPath: entry.relPath,
kind: entry.kind,
driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null,
afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null,
afterSize: after?.byteLength ?? null
});
}
return buildPlan(servers, files);
}
function buildPlan(servers: string[], files: DriftPlanFile[]): DriftPlan {
files.sort((a, b) => `${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`));
return { servers, files, scannedAt: new Date().toISOString(), fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files }))) };
files.sort((a, b) =>
`${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`)
);
return {
servers,
files,
scannedAt: new Date().toISOString(),
fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files })))
};
}
export async function previewDrift(servers?: string[]): Promise<DriftPlan> {
const targets = driftTargets(servers);
const entries: DriftEntry[] = [];
for (const server of targets) entries.push(...await scanServerDrift(server));
for (const server of targets) entries.push(...(await scanServerDrift(server)));
return describePlan(targets, entries);
}
@@ -373,7 +491,10 @@ export interface RunDriftResult {
}
function deploymentIdNow(): string {
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
const stamp = new Date()
.toISOString()
.replace(/[-:]/g, '')
.replace(/\.\d+Z$/, 'Z');
return `dep-${stamp}-drift-${randomUUID().slice(0, 8)}`;
}
@@ -386,47 +507,103 @@ export interface PreparedDriftCampaign {
export async function prepareDriftCampaign(server: string): Promise<PreparedDriftCampaign> {
const servers = driftTargets([server]);
const entries = await scanServerDrift(server);
const files = await scanBatch(entries, async entry => {
const files = await scanBatch(entries, async (entry) => {
const before = await readExistingFile(join(destOfKind(server, entry.kind), entry.relPath));
const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry);
if (entry.driftKind !== 'untracked' && !after) throw new Error(`Cannot resolve promoted content for ${entry.relPath}`);
return { before, after, file: { server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } };
const after =
entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry);
if (entry.driftKind !== 'untracked' && !after)
throw new Error(`Cannot resolve promoted content for ${entry.relPath}`);
return {
before,
after,
file: {
server,
relPath: entry.relPath,
kind: entry.kind,
driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null,
afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null,
afterSize: after?.byteLength ?? null
}
};
});
if (entries.length) await requireServerDirectory(server);
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
return { plan: buildPlan(servers, files.map(({ file }) => file)), files };
if (hasActiveBeta(server))
throw new Error('Active beta owns this server; reconciliation deferred');
return {
plan: buildPlan(
servers,
files.map(({ file }) => file)
),
files
};
}
/** Record the audited plan atomically with a queued rebuild, never a second restart. */
export function createDriftRebuildCampaign(
server: string, operationId: string, incidentId: string | number, prepared: PreparedDriftCampaign
server: string,
operationId: string,
incidentId: string | number,
prepared: PreparedDriftCampaign
): string | null {
if (!prepared.files.length) return null;
if (prepared.plan.servers.length !== 1 || prepared.plan.servers[0] !== server
|| prepared.files.some(({ file }) => file.server !== server)) throw new Error('Drift evidence belongs to another server');
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
if (
prepared.plan.servers.length !== 1 ||
prepared.plan.servers[0] !== server ||
prepared.files.some(({ file }) => file.server !== server)
)
throw new Error('Drift evidence belongs to another server');
if (hasActiveBeta(server))
throw new Error('Active beta owns this server; reconciliation deferred');
return db().transaction(() => {
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'").get(operationId, server)) {
if (
!db()
.prepare(
"SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'"
)
.get(operationId, server)
) {
throw new Error('Drift campaign requires the matching rebuild operation');
}
const id = deploymentIdNow();
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')")
db()
.prepare(
"INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')"
)
.run(id, new Date().toISOString());
const insertFile = db().prepare(`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
for (const { file, before, after } of prepared.files) {
insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated',
file.relPath, file.beforeSha256, file.afterSha256, putFileBlob(before), putFileBlob(after));
insertFile.run(
id,
server,
after === null ? 'removed' : before === null ? 'added' : 'updated',
file.relPath,
file.beforeSha256,
file.afterSha256,
putFileBlob(before),
putFileBlob(after)
);
}
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId);
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)")
db()
.prepare(
`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
VALUES (?, ?, ?, 'pending', ?)`
)
.run(id, server, prepared.files.length, operationId);
db()
.prepare(
"INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)"
)
.run(operationId, id, server);
db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(id, `Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`);
db()
.prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(
id,
`Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`
);
emit({ type: 'deployment-opened', deploymentId: id, action: 'drift', servers: [server] });
return id;
})();
@@ -455,7 +632,12 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
}
// Recheck identities, existing bytes, and replacement bytes before any write.
const plan = input.expectedFingerprint ? await describePlan(scanTargets, perServer.flatMap(server => server.entries)) : null;
const plan = input.expectedFingerprint
? await describePlan(
scanTargets,
perServer.flatMap((server) => server.entries)
)
: null;
if (plan && plan.fingerprint !== input.expectedFingerprint) throw new DriftPlanChanged(plan);
const totalDrift = perServer.reduce((sum, s) => sum + s.entries.length, 0);
if (totalDrift === 0 || dryRun) {
@@ -464,7 +646,9 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
perServer: perServer.map((s) => ({
server: s.server,
removed: s.entries.filter((e) => e.driftKind === 'untracked').map((e) => e.relPath),
restored: s.entries.filter((e) => e.driftKind !== 'untracked').map((e) => e.relPath),
restored: s.entries
.filter((e) => e.driftKind !== 'untracked')
.map((e) => e.relPath),
skipped: []
})),
totalRemoved: perServer.reduce(
@@ -484,9 +668,11 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
// can be inserted as we go.
const depId = deploymentIdNow();
const now = new Date().toISOString();
db().prepare(
`INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)`
).run(depId, now, triggeredBy);
db()
.prepare(
`INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)`
)
.run(depId, now, triggeredBy);
const insertServer = db().prepare(
`INSERT INTO deployment_servers
(deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error)
@@ -511,53 +697,107 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
try {
await requireServerDirectory(s.server);
if (hasActiveBeta(s.server)) {
s.skipped.push(`${entry.relPath}: active beta owns this server; reconciliation deferred`);
s.skipped.push(
`${entry.relPath}: active beta owns this server; reconciliation deferred`
);
continue;
}
const reviewed = plan?.files.find(file => file.server === s.server && file.kind === entry.kind && file.relPath === entry.relPath);
const reviewed = plan?.files.find(
(file) =>
file.server === s.server &&
file.kind === entry.kind &&
file.relPath === entry.relPath
);
if (reviewed) {
const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; });
const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return null;
throw error;
});
if ((current ? sha256Buffer(current) : null) !== reviewed.beforeSha256) {
s.skipped.push(`${entry.relPath}: changed after review; scan and review again`);
s.skipped.push(
`${entry.relPath}: changed after review; scan and review again`
);
continue;
}
}
if (entry.driftKind === 'untracked') {
// Capture the reviewed deletion without touching the live tree.
const content = await readFile(abs);
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
if (hasActiveBeta(s.server))
throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, putFileBlob(content), null);
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) });
const file = insertFile.run(
depId,
s.server,
'removed',
entry.relPath,
sha256Buffer(content),
null,
putFileBlob(content),
null
);
queueFileChange({
sourceKind: 'campaign',
sourceId: depId,
server: s.server,
kind: entry.kind,
relPath: entry.relPath,
content: null,
inventorySource: `drift:${depId}`,
deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content)
});
})();
s.removed.push(entry.relPath);
} else {
// Persist exact replacement bytes so waiting cannot change the plan.
const written = await resolveManagedEntry(s.server, entry);
if (!written) {
s.skipped.push(`${entry.relPath}: missing artifact could not be resolved from the DB`);
s.skipped.push(
`${entry.relPath}: missing artifact could not be resolved from the DB`
);
continue;
}
if (reviewed && sha256Buffer(written) !== reviewed.afterSha256) {
s.skipped.push(`${entry.relPath}: promoted content changed after review; scan and review again`);
s.skipped.push(
`${entry.relPath}: promoted content changed after review; scan and review again`
);
continue;
}
const before = await readFile(abs).catch((err: NodeJS.ErrnoException) => {
if (err.code === 'ENOENT') return null;
throw err;
});
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
if (hasActiveBeta(s.server))
throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath,
before ? sha256Buffer(before) : null, sha256Buffer(written), putFileBlob(before), putFileBlob(written));
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: written,
scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed ? reviewed.beforeSha256 : before ? sha256Buffer(before) : null });
const file = insertFile.run(
depId,
s.server,
before ? 'updated' : 'added',
entry.relPath,
before ? sha256Buffer(before) : null,
sha256Buffer(written),
putFileBlob(before),
putFileBlob(written)
);
queueFileChange({
sourceKind: 'campaign',
sourceId: depId,
server: s.server,
kind: entry.kind,
relPath: entry.relPath,
content: written,
scriptVersionId:
entry.kind === 'config' ? null : (entry.scriptVersionId ?? null),
inventorySource: `drift:${depId}`,
deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed
? reviewed.beforeSha256
: before
? sha256Buffer(before)
: null
});
})();
s.restored.push(entry.relPath);
}
@@ -579,7 +819,10 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
return {
deploymentId: null,
perServer: perServer.map((s) => ({
server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
server: s.server,
removed: s.removed,
restored: s.restored,
skipped: s.skipped
})),
totalRemoved: 0,
totalRestored: 0,
@@ -598,7 +841,10 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
return {
deploymentId: depId,
perServer: perServer.map((s) => ({
server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
server: s.server,
removed: s.removed,
restored: s.restored,
skipped: s.skipped
})),
totalRemoved: perServer.reduce((sum, s) => sum + s.removed.length, 0),
totalRestored: perServer.reduce((sum, s) => sum + s.restored.length, 0),
src/lib/server/fleet/health-observations.ts
+226 -69
View File
@@ -4,7 +4,11 @@ import { db } from '../db/index.js';
import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
import { nativeInstance, type NativeInstance } from '../backends/native.js';
import { nativeHealth } from '../backends/native.js';
import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js';
import {
rconStatusResult,
agentRconStatusResult,
type AgentRconReply
} from '../integrations/rcon.js';
import { withOperationSource } from '../hosts/operation-context.js';
import { getServer } from './servers.js';
import { latestGameSubmission } from './lifecycle-evidence.js';
@@ -14,8 +18,15 @@ import { ONLINE_MAX_MS } from '../../server-freshness.js';
const MAX_AGE_MS = 30_000;
const MAX_SERVERS = 1024;
type Probe = 'inspect' | 'rcon';
interface Observation<T> { result: T; observedAt: string }
interface Identity { key: string; port: number | null; checkedAt: string | null }
interface Observation<T> {
result: T;
observedAt: string;
}
interface Identity {
key: string;
port: number | null;
checkedAt: string | null;
}
interface Entry<T> {
identity: Identity;
startedAt: number;
@@ -28,21 +39,50 @@ export class MonitoringSkippedError extends Error {}
function identity(server: string, probe: Probe, snapshot?: MonitoringSnapshot): Identity {
const row = snapshot ? snapshot.get(server)?.server : getServer(server);
if (!row || row.destroyed_at) throw new MonitoringChangedError('Server is no longer registered; monitoring evidence is unavailable.');
const native = snapshot ? snapshot.get(server)?.native : row.native_instance_id ? nativeInstance(server) : null;
if (row.native_instance_id && (!native || (snapshot ? snapshot.get(server)?.host : getHost(native.host_id))?.connectionState !== 'online')) {
if (!row || row.destroyed_at)
throw new MonitoringChangedError(
'Server is no longer registered; monitoring evidence is unavailable.'
);
const native = snapshot
? snapshot.get(server)?.native
: row.native_instance_id
? nativeInstance(server)
: null;
if (
row.native_instance_id &&
(!native ||
(snapshot ? snapshot.get(server)?.host : getHost(native.host_id))?.connectionState !==
'online')
) {
throw new Error('Host agent is offline; monitoring evidence is unavailable.');
}
// Stop requests also invalidate observations, even though they do not
// establish a new game-telemetry epoch in latestGameSubmission().
const savedLifecycle = snapshot?.get(server)?.lifecycle;
const lifecycle = snapshot ? savedLifecycle ? { id: savedLifecycle.id, action: savedLifecycle.action, submitted_at: savedLifecycle.submitted_at } : null : db().prepare(`SELECT id, action, submitted_at FROM server_deployments
WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) ?? null;
const lifecycle = snapshot
? savedLifecycle
? {
id: savedLifecycle.id,
action: savedLifecycle.action,
submitted_at: savedLifecycle.submitted_at
}
: null
: (db()
.prepare(
`SELECT id, action, submitted_at FROM server_deployments
WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`
)
.get(server) ?? null);
return {
key: JSON.stringify([row.created_at, row.desired_state, snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server), lifecycle,
key: JSON.stringify([
row.created_at,
row.desired_state,
snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server),
lifecycle,
native ? [native.host_id, native.id, native.generation] : null,
probe === 'rcon' ? [row.rcon_host, config.rcon] : null]),
port: probe === 'rcon' ? native?.port ?? row.game_port : null,
probe === 'rcon' ? [row.rcon_host, config.rcon] : null
]),
port: probe === 'rcon' ? (native?.port ?? row.game_port) : null,
checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at
};
}
@@ -58,14 +98,32 @@ function reusable(startedAt: number): boolean {
return age >= 0 && age < MAX_AGE_MS;
}
async function observe<T extends { ok: boolean }>(server: string, fresh: boolean, probe: Probe,
entries: Map<string, Entry<T>>, request: (server: string) => Promise<T>, snapshot?: MonitoringSnapshot, batch = false): Promise<Observation<T>> {
async function observe<T extends { ok: boolean }>(
server: string,
fresh: boolean,
probe: Probe,
entries: Map<string, Entry<T>>,
request: (server: string) => Promise<T>,
snapshot?: MonitoringSnapshot,
batch = false
): Promise<Observation<T>> {
let current: Identity;
try { current = identity(server, probe, snapshot); }
catch (error) { entries.delete(server); throw error; }
try {
current = identity(server, probe, snapshot);
} catch (error) {
entries.delete(server);
throw error;
}
const saved = entries.get(server);
const savedAt = saved?.observation ? Date.parse(saved.observation.observedAt) : saved?.startedAt ?? 0;
if (!fresh && saved && sameIdentity(saved.identity, current) && !newerEvidence(current, savedAt)) {
const savedAt = saved?.observation
? Date.parse(saved.observation.observedAt)
: (saved?.startedAt ?? 0);
if (
!fresh &&
saved &&
sameIdentity(saved.identity, current) &&
!newerEvidence(current, savedAt)
) {
if (saved.pending) return saved.pending;
if (saved.observation && reusable(saved.startedAt)) return saved.observation;
}
@@ -76,22 +134,27 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
entries.delete(server);
entries.set(server, entry);
while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!);
const pending = Promise.resolve().then(() => request(server)).then(result => {
if (!batch) snapshot?.refresh(server);
const latest = identity(server, probe, snapshot);
if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) {
throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.');
}
const observation = { result, observedAt: new Date().toISOString() };
if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) {
entry.identity = latest;
entry.observation = observation;
}
return observation;
}).finally(() => {
entry.pending = undefined;
if (entries.get(server) === entry && !entry.observation) entries.delete(server);
});
const pending = Promise.resolve()
.then(() => request(server))
.then((result) => {
if (!batch) snapshot?.refresh(server);
const latest = identity(server, probe, snapshot);
if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) {
throw new MonitoringChangedError(
'Server changed during monitoring; the previous observation was discarded.'
);
}
const observation = { result, observedAt: new Date().toISOString() };
if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) {
entry.identity = latest;
entry.observation = observation;
}
return observation;
})
.finally(() => {
entry.pending = undefined;
if (entries.get(server) === entry && !entry.observation) entries.delete(server);
});
entry.pending = pending;
return pending;
}
@@ -112,14 +175,19 @@ export function monitoringIdentity(server: string, snapshot?: MonitoringSnapshot
const container = identity(server, 'inspect', snapshot);
const game = identity(server, 'rcon', snapshot);
// Identity includes connection configuration; persist only its fingerprint.
return createHash('sha256').update(JSON.stringify([container.key, game.key, game.port])).digest('hex');
return createHash('sha256')
.update(JSON.stringify([container.key, game.key, game.port]))
.digest('hex');
}
/** Capture ownership before a status pass; publish only evidence for that instance. */
export function monitoringStatusWriter(server: string, snapshot?: MonitoringSnapshot) {
let before: string;
try { before = identity(server, 'inspect', snapshot).key; }
catch { return () => {}; }
try {
before = identity(server, 'inspect', snapshot).key;
} catch {
return () => {};
}
const slot = { snapshot: statuses.get(server)?.snapshot };
statuses.delete(server);
statuses.set(server, slot);
@@ -133,12 +201,17 @@ export function monitoringStatusWriter(server: string, snapshot?: MonitoringSnap
const current = slot.snapshot;
if (current && current.container.observedAt > container.observedAt) return;
slot.snapshot = { identity: monitoringIdentity(server, snapshot), container, game };
} catch { statuses.delete(server); }
} catch {
statuses.delete(server);
}
};
}
/** Latest completed status-monitor pass, with stale or superseded evidence excluded. */
export function monitoringStatus(server: string, snapshot?: MonitoringSnapshot): MonitoringStatus | null {
export function monitoringStatus(
server: string,
snapshot?: MonitoringSnapshot
): MonitoringStatus | null {
const saved = statuses.get(server)?.snapshot;
if (!saved) return null;
const fresh = (at: string) => {
@@ -146,43 +219,94 @@ export function monitoringStatus(server: string, snapshot?: MonitoringSnapshot):
return age >= 0 && age <= ONLINE_MAX_MS;
};
try {
if (saved.identity !== monitoringIdentity(server, snapshot)
|| !fresh(saved.container.observedAt)
|| newerEvidence(identity(server, 'inspect', snapshot), Date.parse(saved.container.observedAt))) return null;
const game = saved.game && fresh(saved.game.observedAt)
&& !newerEvidence(identity(server, 'rcon', snapshot), Date.parse(saved.game.observedAt)) ? saved.game : undefined;
if (
saved.identity !== monitoringIdentity(server, snapshot) ||
!fresh(saved.container.observedAt) ||
newerEvidence(
identity(server, 'inspect', snapshot),
Date.parse(saved.container.observedAt)
)
)
return null;
const game =
saved.game &&
fresh(saved.game.observedAt) &&
!newerEvidence(identity(server, 'rcon', snapshot), Date.parse(saved.game.observedAt))
? saved.game
: undefined;
return { ...saved, game };
} catch { return null; }
} catch {
return null;
}
}
/** Keep the actual sample time: a later heartbeat must not renew old evidence. */
export function heartbeatInspection(server: string, snapshot?: MonitoringSnapshot): Observation<Awaited<ReturnType<typeof nativeHealth>>> | null {
export function heartbeatInspection(
server: string,
snapshot?: MonitoringSnapshot
): Observation<Awaited<ReturnType<typeof nativeHealth>>> | null {
const row = snapshot ? snapshot.get(server)?.server : getServer(server);
if (!row?.native_instance_id || !row.host_id) return null;
const host = snapshot ? snapshot.get(server)?.host : getHost(row.host_id);
if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state')) return null;
const state = host.inventory.instanceStates?.find(state => state.instanceId === row.native_instance_id && state.generation === row.instance_generation);
if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state'))
return null;
const state = host.inventory.instanceStates?.find(
(state) =>
state.instanceId === row.native_instance_id &&
state.generation === row.instance_generation
);
if (!state) return null;
const at = Date.parse(state.observedAt);
const lifecycle = snapshot ? { at: snapshot.get(server)?.lifecycleAt } : db().prepare(`SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at
FROM server_deployments WHERE server=? AND server_deleted_at IS NULL`).get(server) as { at: string | null };
if (!reusable(at) || at < Date.parse(row.created_at) || at < Date.parse(lifecycle.at ?? '')
|| at < Date.parse((snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server)) ?? '') || at < Date.parse(row.container_checked_at ?? '')) return null;
return { observedAt: new Date(at).toISOString(), result: state.healthy
? { ok: true, state: state.state } : { ok: false, state: state.state, reason: `Container ${state.state}` } };
const lifecycle = snapshot
? { at: snapshot.get(server)?.lifecycleAt }
: (db()
.prepare(
`SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at
FROM server_deployments WHERE server=? AND server_deleted_at IS NULL`
)
.get(server) as { at: string | null });
if (
!reusable(at) ||
at < Date.parse(row.created_at) ||
at < Date.parse(lifecycle.at ?? '') ||
at <
Date.parse(
(snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server)) ?? ''
) ||
at < Date.parse(row.container_checked_at ?? '')
)
return null;
return {
observedAt: new Date(at).toISOString(),
result: state.healthy
? { ok: true, state: state.state }
: { ok: false, state: state.state, reason: `Container ${state.state}` }
};
}
type RconObservation = Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
/** Prime a whole host before the per-host queue admits just its first two servers. */
export function prepareMonitoringRcon(servers: string[], snapshot?: MonitoringSnapshot): Map<string, () => Promise<RconObservation>> {
export function prepareMonitoringRcon(
servers: string[],
snapshot?: MonitoringSnapshot
): Map<string, () => Promise<RconObservation>> {
const probes = new Map<string, () => Promise<RconObservation>>();
const batches = new Map<string, { instances: NativeInstance[]; result: Promise<{ results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> }> }>();
const batches = new Map<
string,
{
instances: NativeInstance[];
result: Promise<{
results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }>;
}>;
}
>();
for (const server of servers) {
const instance = snapshot ? snapshot.get(server)?.native : nativeInstance(server);
if (!instance) continue;
const host = snapshot ? snapshot.get(server)?.host : getHost(instance.host_id);
if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch')) continue;
if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch'))
continue;
const container = heartbeatInspection(server, snapshot);
if (container && !container.result.ok && container.result.state !== 'unknown') continue;
const before = identity(server, 'rcon', snapshot);
@@ -193,40 +317,73 @@ export function prepareMonitoringRcon(servers: string[], snapshot?: MonitoringSn
// observe() invokes each request in the same microtask turn;
// enqueue afterward so cached or shared probes need no batch slot.
const result = Promise.resolve().then(async () => {
const operation = withOperationSource('Status monitor', () => queueHostOperation(instance.host_id, 'rcon.status-batch', {
instances: instances.map(value => ({ instanceId: value.id, generation: value.generation }))
}));
const operation = withOperationSource('Status monitor', () =>
queueHostOperation(instance.host_id, 'rcon.status-batch', {
instances: instances.map((value) => ({
instanceId: value.id,
generation: value.generation
}))
})
);
// Allow delivery and receipt overhead beyond the agent's 20-second budget.
const receipt = await waitHostOperationStatus(operation.id, 25_000);
snapshot?.refresh();
if (receipt.state !== 'completed') throw new Error(receipt.error ?? 'Host status batch did not complete.');
return receipt.result as { results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> };
if (receipt.state !== 'completed')
throw new Error(receipt.error ?? 'Host status batch did not complete.');
return receipt.result as {
results: Array<{
instanceId: string;
generation: number;
reply: AgentRconReply;
}>;
};
});
batch = { instances, result }; batches.set(instance.host_id, batch);
batch = { instances, result };
batches.set(instance.host_id, batch);
}
batch.instances.push(instance);
const endpoint = { host: `agent:${instance.host_id}`, port: instance.port };
try {
const receipt = await batch.result;
const replies = receipt.results.filter(value => value.instanceId === instance.id && value.generation === instance.generation);
if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string') throw new Error('Host status batch omitted a valid instance result.');
if (replies[0].reply.reason === 'deadline') throw new MonitoringSkippedError('Host status batch budget expired before this server was checked.');
const replies = receipt.results.filter(
(value) =>
value.instanceId === instance.id && value.generation === instance.generation
);
if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string')
throw new Error('Host status batch omitted a valid instance result.');
if (replies[0].reply.reason === 'deadline')
throw new MonitoringSkippedError(
'Host status batch budget expired before this server was checked.'
);
return agentRconStatusResult(replies[0].reply, endpoint);
} catch (error) {
if (error instanceof MonitoringSkippedError) throw error;
return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message };
return {
ok: false as const,
endpoint,
reason: 'timeout' as const,
detail: (error as Error).message
};
}
};
// Observe before dispatch, including servers still waiting for their
// queue slot. Consume failures too, so a failed batch is never retried
// once per server or left as an unhandled rejected promise.
const pending = observe(server, false, 'rcon', rcon, request, snapshot, true).then(value => ({ value }), error => ({ error }));
const pending = observe(server, false, 'rcon', rcon, request, snapshot, true).then(
(value) => ({ value }),
(error) => ({ error })
);
probes.set(server, async () => {
const result = await pending;
if ('error' in result) throw result.error;
const current = identity(server, 'rcon', snapshot);
if (!sameIdentity(before, current) || newerEvidence(current, Date.parse(result.value.observedAt))) {
throw new MonitoringChangedError('Server changed before consuming the host status batch.');
if (
!sameIdentity(before, current) ||
newerEvidence(current, Date.parse(result.value.observedAt))
) {
throw new MonitoringChangedError(
'Server changed before consuming the host status batch.'
);
}
return result.value;
});
src/lib/server/fleet/health.ts
+7 -2
View File
@@ -9,9 +9,14 @@ const STALE_MAX_MS = 120_000;
* `stale` on a blocking RCON failure or if the last reply is >90-120s old.
* `online` if fresh, including the allowance for polling and delivery.
*/
export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
export function classifyHealth(
lastSeenAt: string | null,
lastError: string | null,
containerState?: string | null
): Health {
if (containerState === 'starting' || containerState === 'restarting') return 'starting';
if (containerState && containerState !== 'running' && containerState !== 'unknown') return 'offline';
if (containerState && containerState !== 'running' && containerState !== 'unknown')
return 'offline';
if (!lastSeenAt) return 'offline';
const age = Date.now() - new Date(lastSeenAt).getTime();
if (freshStatusTimestamp(lastSeenAt, Date.now())) {
src/lib/server/fleet/house-party.ts
+17 -5
View File
@@ -2,23 +2,35 @@ import { db } from '../db/index.js';
import { destroyServer } from '../provisioning/index.js';
export function housePartyTargets(): string[] {
return (db().prepare(`
return (
db()
.prepare(
`
SELECT name FROM servers WHERE destroyed_at IS NULL
UNION
SELECT server AS name FROM server_deployments
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
ORDER BY name
`).all() as { name: string }[]).map(row => row.name);
`
)
.all() as { name: string }[]
).map((row) => row.name);
}
export function queueHouseParty(expectedServers: string[]) {
return db().transaction(() => {
const servers = housePartyTargets();
const expected = [...expectedServers].sort();
if (servers.length !== expected.length || servers.some((name, index) => name !== expected[index])) {
return { ok: false as const, error: 'The fleet changed. Refresh the preview and review the servers again.' };
if (
servers.length !== expected.length ||
servers.some((name, index) => name !== expected[index])
) {
return {
ok: false as const,
error: 'The fleet changed. Refresh the preview and review the servers again.'
};
}
const deployments = servers.map(server => {
const deployments = servers.map((server) => {
const result = destroyServer(server, 'house-party-protocol');
if (!result.ok) throw new Error(result.message);
return { server, deploymentId: result.deploymentId! };
src/lib/server/fleet/inventory.ts
+3 -1
View File
@@ -36,7 +36,9 @@ export function recordScript(
}
export function removeScript(server: string, relPath: string): void {
db().prepare('DELETE FROM server_scripts WHERE server = ? AND rel_path = ?').run(server, relPath);
db()
.prepare('DELETE FROM server_scripts WHERE server = ? AND rel_path = ?')
.run(server, relPath);
}
export function getServerScript(server: string, relPath: string): ServerScriptRow | undefined {
src/lib/server/fleet/launch-recipes.ts
+248 -68
View File
@@ -1,12 +1,20 @@
import { packFileJson, unpackFileJson } from '../db/file-blobs.js';
import { randomUUID } from 'node:crypto';
import type { BlueprintSummary, LaunchRecipeSummary, LaunchFileSummary } from '$lib/deployment-plan.js';
import type {
BlueprintSummary,
LaunchRecipeSummary,
LaunchFileSummary
} from '$lib/deployment-plan.js';
import { config, configFileForGame } from '../config.js';
import { db } from '../db/index.js';
import { zombieMapBySlug, type ZombieGame } from './map-aliases.js';
import { promotedArtifactMetadata, promotedArtifacts } from './promoted-artifacts.js';
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
import { getBaselineForScope, getVersionMetadata, listPromotedOverlaysForScope } from './scripts.js';
import {
getBaselineForScope,
getVersionMetadata,
listPromotedOverlaysForScope
} from './scripts.js';
import { sha256Buffer } from './files.js';
export interface LaunchRecipePlan {
@@ -28,165 +36,337 @@ export interface LaunchRecipe extends LaunchRecipePlan {
files: (LaunchFileSummary & { content: string })[];
}
export interface BlueprintEntry { count: number; recipe: LaunchRecipePlan }
interface BlueprintSelection { count: number; game: ZombieGame; mapSlug: string }
interface BlueprintRow { id: string; name: string; description: string; created_at: string; recipe: string }
export interface BlueprintEntry {
count: number;
recipe: LaunchRecipePlan;
}
interface BlueprintSelection {
count: number;
game: ZombieGame;
mapSlug: string;
}
interface BlueprintRow {
id: string;
name: string;
description: string;
created_at: string;
recipe: string;
}
/** Legacy snapshots provide the lineup only; their file versions never govern a new launch. */
function readEntries(raw: string): BlueprintSelection[] {
const saved = JSON.parse(raw) as LaunchRecipe | { format: 2; entries: BlueprintEntry[] }
const saved = JSON.parse(raw) as
| LaunchRecipe
| { format: 2; entries: BlueprintEntry[] }
| { format: 3; entries: BlueprintSelection[] };
if (saved.format === 3) return saved.entries;
const entries = saved.format === 2 ? saved.entries : [{ count: 1, recipe: saved }];
return entries.map(({ count, recipe }) => ({ count, game: recipe.game, mapSlug: recipe.mapSlug }));
return entries.map(({ count, recipe }) => ({
count,
game: recipe.game,
mapSlug: recipe.mapSlug
}));
}
function summarizeBlueprint(row: BlueprintRow): BlueprintSummary {
const entries = readEntries(row.recipe).map(entry => ({ ...entry,
mapName: zombieMapBySlug(entry.game, entry.mapSlug)?.friendly ?? entry.mapSlug }));
return { id: row.id, name: row.name, description: row.description, createdAt: row.created_at,
entries, totalServers: entries.reduce((total, entry) => total + entry.count, 0) };
const entries = readEntries(row.recipe).map((entry) => ({
...entry,
mapName: zombieMapBySlug(entry.game, entry.mapSlug)?.friendly ?? entry.mapSlug
}));
return {
id: row.id,
name: row.name,
description: row.description,
createdAt: row.created_at,
entries,
totalServers: entries.reduce((total, entry) => total + entry.count, 0)
};
}
export function blueprintEntries(id: string): { entries: BlueprintEntry[]; blueprintName: string } {
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as BlueprintRow | undefined;
if (!row) throw new Error('This collection is no longer available. Choose another starting point.');
const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipePlan(game, mapSlug) }));
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as
BlueprintRow | undefined;
if (!row)
throw new Error('This collection is no longer available. Choose another starting point.');
const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({
count,
recipe: captureRecipePlan(game, mapSlug)
}));
return { entries, blueprintName: row.name };
}
function buildEntries(input: unknown): BlueprintSelection[] {
if (!Array.isArray(input) || input.length < 1 || input.length > 40) throw new Error('Add between 1 and 40 map selections.');
if (!Array.isArray(input) || input.length < 1 || input.length > 40)
throw new Error('Add between 1 and 40 map selections.');
const seen = new Set<string>();
const entries = input.map(entry => {
if (!entry || (entry.game !== 't5' && entry.game !== 't6') || typeof entry.mapSlug !== 'string'
|| !zombieMapBySlug(entry.game, entry.mapSlug)) throw new Error('Each selection needs a supported game and map.');
if (!Number.isSafeInteger(entry.count) || entry.count < 1 || entry.count > 15) throw new Error('Choose 1–15 lobbies for each map.');
const entries = input.map((entry) => {
if (
!entry ||
(entry.game !== 't5' && entry.game !== 't6') ||
typeof entry.mapSlug !== 'string' ||
!zombieMapBySlug(entry.game, entry.mapSlug)
)
throw new Error('Each selection needs a supported game and map.');
if (!Number.isSafeInteger(entry.count) || entry.count < 1 || entry.count > 15)
throw new Error('Choose 1–15 lobbies for each map.');
const key = `${entry.game}:${entry.mapSlug}`;
if (seen.has(key)) throw new Error('Each game/map can appear once. Increase its lobby count instead.');
if (seen.has(key))
throw new Error('Each game/map can appear once. Increase its lobby count instead.');
seen.add(key);
return { count: entry.count as number, game: entry.game as ZombieGame, mapSlug: entry.mapSlug as string };
return {
count: entry.count as number,
game: entry.game as ZombieGame,
mapSlug: entry.mapSlug as string
};
});
if (entries.reduce((total, entry) => total + entry.count, 0) > 100) throw new Error('A collection can launch at most 100 servers.');
if (entries.reduce((total, entry) => total + entry.count, 0) > 100)
throw new Error('A collection can launch at most 100 servers.');
return entries;
}
export function captureRecipe(game: ZombieGame, mapSlug: string, rconPassword?: string): LaunchRecipe {
export function captureRecipe(
game: ZombieGame,
mapSlug: string,
rconPassword?: string
): LaunchRecipe {
return buildRecipe(game, mapSlug, true, rconPassword);
}
/** Preview and admission keep file references; execution captures the current bytes. */
export function captureRecipePlan(game: ZombieGame, mapSlug: string, rconPassword?: string): LaunchRecipePlan {
export function captureRecipePlan(
game: ZombieGame,
mapSlug: string,
rconPassword?: string
): LaunchRecipePlan {
return buildRecipe(game, mapSlug, false, rconPassword);
}
function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: true, rconPassword?: string): LaunchRecipe;
function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: false, rconPassword?: string): LaunchRecipePlan;
function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean, rconPassword?: string): LaunchRecipePlan {
function buildRecipe(
game: ZombieGame,
mapSlug: string,
includeContent: true,
rconPassword?: string
): LaunchRecipe;
function buildRecipe(
game: ZombieGame,
mapSlug: string,
includeContent: false,
rconPassword?: string
): LaunchRecipePlan;
function buildRecipe(
game: ZombieGame,
mapSlug: string,
includeContent: boolean,
rconPassword?: string
): LaunchRecipePlan {
const map = zombieMapBySlug(game, mapSlug);
if (!map) throw new Error('Choose a supported game and map.');
const metadata = ({ kind, script, version }: ReturnType<typeof promotedArtifactMetadata>[number]): LaunchFileSummary => ({
name: script.name, kind, path: script.target_rel_path, versionId: version.id,
version: version.version, sha256: version.sha256, size: version.size
const metadata = ({
kind,
script,
version
}: ReturnType<typeof promotedArtifactMetadata>[number]): LaunchFileSummary => ({
name: script.name,
kind,
path: script.target_rel_path,
versionId: version.id,
version: version.version,
sha256: version.sha256,
size: version.size
});
const files: LaunchRecipePlan['files'] = includeContent
? promotedArtifacts(game, map.canonical).map(artifact => ({ ...metadata(artifact), content: artifact.version.content.toString('base64') }))
? promotedArtifacts(game, map.canonical).map((artifact) => ({
...metadata(artifact),
content: artifact.version.content.toString('base64')
}))
: promotedArtifactMetadata(game, map.canonical).map(metadata);
const addConfig = (name: string, path: string, content: Buffer) => files.push({
name, path, kind: 'config', versionId: null, version: null,
sha256: sha256Buffer(content), size: content.length, ...(includeContent ? { content: content.toString('base64') } : {})
});
const addConfig = (name: string, path: string, content: Buffer) =>
files.push({
name,
path,
kind: 'config',
versionId: null,
version: null,
sha256: sha256Buffer(content),
size: content.length,
...(includeContent ? { content: content.toString('base64') } : {})
});
const composed = getPromotedServerConfig(game, map.canonical, map.slug, rconPassword);
let configSource = 'Plutainer defaults, seeded on first boot';
if (composed) {
const baseline = getBaselineForScope('game', game)!;
const layers = [baseline, ...listPromotedOverlaysForScope('game', game), ...listPromotedOverlaysForScope('map', map.canonical)];
configSource = layers.map(layer => `${layer.name} v${getVersionMetadata(layer.current_version_id!)!.version}`).join(' + ');
const layers = [
baseline,
...listPromotedOverlaysForScope('game', game),
...listPromotedOverlaysForScope('map', map.canonical)
];
configSource = layers
.map(
(layer) =>
`${layer.name} v${getVersionMetadata(layer.current_version_id!)!.version}`
)
.join(' + ');
addConfig('Composed server configuration', configFileForGame(game), composed);
}
addConfig('Selected map', MAP_CONFIG_FILE, getServerMapConfig(game, map.canonical, map.slug)!);
return { format: 1, game, mapSlug, mapName: map.friendly, expectedMap: map.canonical,
capturedAt: new Date().toISOString(), image: config.defaultRuntimeImage,
configFile: configFileForGame(game), configSource, files };
return {
format: 1,
game,
mapSlug,
mapName: map.friendly,
expectedMap: map.canonical,
capturedAt: new Date().toISOString(),
image: config.defaultRuntimeImage,
configFile: configFileForGame(game),
configSource,
files
};
}
export function summarizeRecipe(recipe: LaunchRecipePlan, fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe)))): LaunchRecipeSummary {
export function summarizeRecipe(
recipe: LaunchRecipePlan,
fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe)))
): LaunchRecipeSummary {
const { format: _format, files, ...summary } = recipe;
return { ...summary, files: files.map(({ content: _content, ...file }) => file),
fingerprint };
return { ...summary, files: files.map(({ content: _content, ...file }) => file), fingerprint };
}
/** Keep the small detail response beside the immutable attempted file bytes. */
export function recipeStorage(recipe: LaunchRecipePlan): { recipe: string; recipeSummary: string } {
const encoded = JSON.stringify(recipe);
return { recipe: JSON.stringify(packFileJson(recipe).value), recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) };
return {
recipe: JSON.stringify(packFileJson(recipe).value),
recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded))))
};
}
export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
const row = db().prepare('SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
export function recipeSummaryForDeployment(
id: string
): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
const row = db()
.prepare(
'SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?'
)
.get(id) as { recipe_summary: string | null; blueprint_name: string | null } | undefined;
if (!row) return null;
if (row.recipe_summary !== null) return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name };
if (row.recipe_summary !== null)
return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name };
// Old snapshots remain readable. Populate on first access, without making
// controller startup scan and decode every retained plugin in its history.
const launch = recipeForDeployment(id)!;
const summary = summarizeRecipe(launch.recipe);
db().prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?').run(JSON.stringify(summary), id);
db()
.prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?')
.run(JSON.stringify(summary), id);
return { recipe: summary, blueprintName: row.blueprint_name };
}
export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
const row = db().prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
export function recipeForDeployment(
id: string
): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
const row = db()
.prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
.get(id) as { recipe: string; blueprint_name: string | null } | undefined;
return row ? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name } : null;
return row
? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name }
: null;
}
export function blueprintRecipe(id: string): { recipe: LaunchRecipePlan; blueprintName: string } {
const { entries, blueprintName } = blueprintEntries(id);
if (entries.length !== 1 || entries[0].count !== 1) throw new Error('This collection launches multiple servers. Use the collection preview.');
if (entries.length !== 1 || entries[0].count !== 1)
throw new Error('This collection launches multiple servers. Use the collection preview.');
return { recipe: entries[0].recipe, blueprintName };
}
export function listBlueprints(): BlueprintSummary[] {
return (db().prepare('SELECT * FROM server_blueprints ORDER BY name COLLATE NOCASE').all() as BlueprintRow[]).map(summarizeBlueprint);
return (
db()
.prepare('SELECT * FROM server_blueprints ORDER BY name COLLATE NOCASE')
.all() as BlueprintRow[]
).map(summarizeBlueprint);
}
function blueprintName(input: { name: unknown; description?: unknown }, exceptId = ''): string {
if (typeof input.name !== 'string' || !input.name.trim() || input.name.trim().length > 80) throw new Error('Give the collection a name of 1–80 characters.');
if (input.description !== undefined && (typeof input.description !== 'string' || input.description.length > 500)) throw new Error('Keep the description to 500 characters.');
if (db().prepare('SELECT id FROM server_blueprints WHERE name = ? COLLATE NOCASE AND id != ?').get(input.name.trim(), exceptId)) throw new Error('A collection with this name already exists. Choose a new name.');
if (typeof input.name !== 'string' || !input.name.trim() || input.name.trim().length > 80)
throw new Error('Give the collection a name of 1–80 characters.');
if (
input.description !== undefined &&
(typeof input.description !== 'string' || input.description.length > 500)
)
throw new Error('Keep the description to 500 characters.');
if (
db()
.prepare('SELECT id FROM server_blueprints WHERE name = ? COLLATE NOCASE AND id != ?')
.get(input.name.trim(), exceptId)
)
throw new Error('A collection with this name already exists. Choose a new name.');
return input.name.trim();
}
export function saveBlueprint(input: { name: unknown; description?: unknown; previewId?: unknown; deploymentId?: unknown; entries?: unknown }): BlueprintSummary {
export function saveBlueprint(input: {
name: unknown;
description?: unknown;
previewId?: unknown;
deploymentId?: unknown;
entries?: unknown;
}): BlueprintSummary {
const name = blueprintName(input);
if ([input.previewId, input.deploymentId, input.entries].filter(value => value !== undefined).length !== 1) throw new Error('Choose exactly one launch preview, recorded deployment, or set of maps.');
if (
[input.previewId, input.deploymentId, input.entries].filter((value) => value !== undefined)
.length !== 1
)
throw new Error('Choose exactly one launch preview, recorded deployment, or set of maps.');
let recipe: LaunchRecipePlan | undefined;
if (typeof input.previewId === 'string') {
const row = db().prepare('SELECT recipe FROM server_launch_previews WHERE id = ? AND expires_at > ?')
const row = db()
.prepare('SELECT recipe FROM server_launch_previews WHERE id = ? AND expires_at > ?')
.get(input.previewId, new Date().toISOString()) as { recipe: string } | undefined;
if (row) recipe = JSON.parse(row.recipe);
} else if (typeof input.deploymentId === 'string') recipe = recipeForDeployment(input.deploymentId)?.recipe;
if (!recipe && input.entries === undefined) throw new Error('The launch recipe is unavailable or the preview expired. Generate a fresh preview.');
const entries = input.entries !== undefined ? buildEntries(input.entries)
: buildEntries([{ count: 1, game: recipe!.game, mapSlug: recipe!.mapSlug }]);
} else if (typeof input.deploymentId === 'string')
recipe = recipeForDeployment(input.deploymentId)?.recipe;
if (!recipe && input.entries === undefined)
throw new Error(
'The launch recipe is unavailable or the preview expired. Generate a fresh preview.'
);
const entries =
input.entries !== undefined
? buildEntries(input.entries)
: buildEntries([{ count: 1, game: recipe!.game, mapSlug: recipe!.mapSlug }]);
const id = randomUUID();
const createdAt = new Date().toISOString();
const saved = JSON.stringify({ format: 3, entries });
db().prepare('INSERT INTO server_blueprints (id, name, description, created_at, recipe) VALUES (?, ?, ?, ?, ?)')
db()
.prepare(
'INSERT INTO server_blueprints (id, name, description, created_at, recipe) VALUES (?, ?, ?, ?, ?)'
)
.run(id, name, input.description ?? '', createdAt, saved);
return summarizeBlueprint({ id, name, description: (input.description as string) ?? '', created_at: createdAt, recipe: saved });
return summarizeBlueprint({
id,
name,
description: (input.description as string) ?? '',
created_at: createdAt,
recipe: saved
});
}
export function updateBlueprint(id: string, input: { name: unknown; description?: unknown; entries?: unknown }): BlueprintSummary {
export function updateBlueprint(
id: string,
input: { name: unknown; description?: unknown; entries?: unknown }
): BlueprintSummary {
return db().transaction(() => {
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as BlueprintRow | undefined;
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as
BlueprintRow | undefined;
if (!row) throw new Error('Collection not found.');
const name = blueprintName(input, id);
const entries = buildEntries(input.entries);
const recipe = JSON.stringify({ format: 3, entries });
const description = (input.description as string) ?? '';
db().prepare('UPDATE server_blueprints SET name = ?, description = ?, recipe = ? WHERE id = ?').run(name, description, recipe, id);
db()
.prepare(
'UPDATE server_blueprints SET name = ?, description = ?, recipe = ? WHERE id = ?'
)
.run(name, description, recipe, id);
return summarizeBlueprint({ ...row, name, description, recipe });
})();
}
src/lib/server/fleet/leaderboard-ingest.ts
+133 -37
View File
@@ -20,7 +20,9 @@ interface Participation {
}
function mapIdentity(handle: Database.Database, report: GameTelemetry) {
const server = handle.prepare('SELECT game, map, map_slug FROM servers WHERE name = ?').get(report.server) as
const server = handle
.prepare('SELECT game, map, map_slug FROM servers WHERE name = ?')
.get(report.server) as
{ game: string | null; map: string | null; map_slug: string | null } | undefined;
const parsed = parseServerName(report.server);
const game = (server?.game ?? parsed?.game ?? 'unknown').toLowerCase();
@@ -28,15 +30,22 @@ function mapIdentity(handle: Database.Database, report: GameTelemetry) {
// A shared engine map (notably zm_transit) is not enough to identify T6's
// location/ruleset. Use the registered selection only when it matches.
const parsedSlug = parsed?.mapCanonical === map ? parsed.mapSlug : undefined;
const slug = server?.map === map ? server.map_slug ?? parsedSlug : parsedSlug;
const candidates = zombieMaps(game).filter(entry => entry.canonical === map);
const selected = game === 't6' && candidates.length > 1 && !slug
? undefined : selectedZombieMap(game, map, slug);
const variant = game === 't6' ? selected?.slug ?? slug ?? 'unspecified' : '';
const slug = server?.map === map ? (server.map_slug ?? parsedSlug) : parsedSlug;
const candidates = zombieMaps(game).filter((entry) => entry.canonical === map);
const selected =
game === 't6' && candidates.length > 1 && !slug
? undefined
: selectedZombieMap(game, map, slug);
const variant = game === 't6' ? (selected?.slug ?? slug ?? 'unspecified') : '';
return {
game, map,
game,
map,
key: [game, map, ...(variant ? [variant] : [])].join(':'),
name: selected?.friendly ?? (game === 't6' && candidates.length > 1 ? `${map} (${slug ?? 'unspecified variant'})` : map)
name:
selected?.friendly ??
(game === 't6' && candidates.length > 1
? `${map} (${slug ?? 'unspecified variant'})`
: map)
};
}
@@ -44,48 +53,92 @@ function fingerprint(report: GameTelemetry): string {
// Receipt timestamps and roster ordering do not identify new gameplay.
const { receivedAt: _receivedAt, timing: _timing, ...event } = report;
const players = [...event.players].sort((a, b) => a.guid.localeCompare(b.guid));
return createHash('sha256').update(JSON.stringify({ ...event, players })).digest('hex');
return createHash('sha256')
.update(JSON.stringify({ ...event, players }))
.digest('hex');
}
/** Called inside the same transaction as the latest scoreboard snapshot. */
export function recordLeaderboardTelemetry(handle: Database.Database, report: GameTelemetry): void {
const identity = mapIdentity(handle, report);
const signature = fingerprint(report);
let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1')
let match = handle
.prepare(
'SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1'
)
.get(report.server) as Match | undefined;
if (match?.last_fingerprint === signature && match.map_key === identity.key) return;
const isStart = report.event === 'game_start';
const round = isStart ? 0 : report.round;
const newMatch = !match || match.map_key !== identity.key
|| (isStart && (match.status !== 'active' || match.last_event !== 'game_start'))
|| (!isStart && round < match.highest_round)
|| ((report.event === 'round_change' || report.event === 'heartbeat') && match.status !== 'active');
const newMatch =
!match ||
match.map_key !== identity.key ||
(isStart && (match.status !== 'active' || match.last_event !== 'game_start')) ||
(!isStart && round < match.highest_round) ||
((report.event === 'round_change' || report.event === 'heartbeat') &&
match.status !== 'active');
if (newMatch) {
if (match?.status === 'active') {
handle.prepare("UPDATE leaderboard_matches SET status = 'interrupted', ended_at = ? WHERE id = ?")
handle
.prepare(
"UPDATE leaderboard_matches SET status = 'interrupted', ended_at = ? WHERE id = ?"
)
.run(report.receivedAt, match.id);
}
const inserted = handle.prepare(`INSERT INTO leaderboard_matches
const inserted = handle
.prepare(
`INSERT INTO leaderboard_matches
(server, game, map, map_key, map_name, started_at, last_received_at, start_observed, last_event, last_fingerprint)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
.run(report.server, identity.game, identity.map, identity.key, identity.name,
report.receivedAt, report.receivedAt, Number(isStart), report.event, signature);
match = { id: Number(inserted.lastInsertRowid), game: identity.game, map_key: identity.key,
status: 'active', highest_round: 0, last_event: report.event, last_fingerprint: signature };
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(
report.server,
identity.game,
identity.map,
identity.key,
identity.name,
report.receivedAt,
report.receivedAt,
Number(isStart),
report.event,
signature
);
match = {
id: Number(inserted.lastInsertRowid),
game: identity.game,
map_key: identity.key,
status: 'active',
highest_round: 0,
last_event: report.event,
last_fingerprint: signature
};
}
if (!match) return;
if (isStart) {
handle.prepare('UPDATE leaderboard_matches SET start_players = COALESCE(start_players, ?) WHERE id = ?')
handle
.prepare(
'UPDATE leaderboard_matches SET start_players = COALESCE(start_players, ?) WHERE id = ?'
)
.run(report.players.length, match.id);
}
const previous = new Map((handle.prepare('SELECT guid, last_round, present FROM leaderboard_match_players WHERE match_id = ?')
.all(match.id) as Participation[]).map(player => [player.guid, player]));
handle.prepare('UPDATE leaderboard_match_players SET present = 0 WHERE match_id = ?').run(match.id);
const saveIdentity = handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
const previous = new Map(
(
handle
.prepare(
'SELECT guid, last_round, present FROM leaderboard_match_players WHERE match_id = ?'
)
.all(match.id) as Participation[]
).map((player) => [player.guid, player])
);
handle
.prepare('UPDATE leaderboard_match_players SET present = 0 WHERE match_id = ?')
.run(match.id);
const saveIdentity =
handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET
name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END,
last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`);
@@ -108,33 +161,76 @@ export function recordLeaderboardTelemetry(handle: Database.Database, report: Ga
const completedRounds = Math.max(0, round - 1);
for (const player of report.players) {
const prior = previous.get(player.guid);
const survived = !isStart && prior?.present ? Math.max(0, completedRounds - prior.last_round) : 0;
saveIdentity.run(identity.game, player.guid, player.name, report.receivedAt, report.receivedAt);
const survived =
!isStart && prior?.present ? Math.max(0, completedRounds - prior.last_round) : 0;
saveIdentity.run(
identity.game,
player.guid,
player.name,
report.receivedAt,
report.receivedAt
);
const stats = 'kills' in player ? player : null;
savePlayer.run(match.id, player.guid, round, survived,
stats?.kills ?? 0, stats?.headshots ?? 0, stats?.downs ?? 0, stats?.revives ?? 0, stats?.score ?? 0,
completedRounds, Number(!isStart), Number(report.event === 'game_end'), report.receivedAt, report.receivedAt, round, Number(isStart));
savePlayer.run(
match.id,
player.guid,
round,
survived,
stats?.kills ?? 0,
stats?.headshots ?? 0,
stats?.downs ?? 0,
stats?.revives ?? 0,
stats?.score ?? 0,
completedRounds,
Number(!isStart),
Number(report.event === 'game_end'),
report.receivedAt,
report.receivedAt,
round,
Number(isStart)
);
}
handle.prepare(`UPDATE leaderboard_matches SET highest_round = MAX(highest_round, ?),
handle
.prepare(
`UPDATE leaderboard_matches SET highest_round = MAX(highest_round, ?),
last_received_at = ?, last_event = ?, last_fingerprint = ?,
status = CASE WHEN ? = 'game_end' THEN 'completed' ELSE status END,
ended_at = CASE WHEN ? = 'game_end' THEN ? ELSE ended_at END WHERE id = ?`)
.run(round, report.receivedAt, report.event, signature, report.event, report.event, report.receivedAt, match.id);
ended_at = CASE WHEN ? = 'game_end' THEN ? ELSE ended_at END WHERE id = ?`
)
.run(
round,
report.receivedAt,
report.event,
signature,
report.event,
report.event,
report.receivedAt,
match.id
);
}
/** Seed the single surviving scoreboard per server once; older games are gone. */
export function backfillLeaderboard(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'leaderboard_v1'").get()) return;
handle.transaction(() => {
const reports = handle.prepare(`SELECT payload FROM server_telemetry WHERE payload IS NOT NULL ORDER BY received_at, server`)
const reports = handle
.prepare(
`SELECT payload FROM server_telemetry WHERE payload IS NOT NULL ORDER BY received_at, server`
)
.all() as Array<{ payload: string }>;
for (const row of reports) {
let report: GameTelemetry;
try {
const stored = JSON.parse(row.payload);
if (typeof stored.receivedAt !== 'string' || !Number.isFinite(Date.parse(stored.receivedAt))) continue;
if (
typeof stored.receivedAt !== 'string' ||
!Number.isFinite(Date.parse(stored.receivedAt))
)
continue;
report = { ...parseGameEvent(stored), receivedAt: stored.receivedAt };
} catch { continue; } // Legacy malformed snapshots cannot establish player records.
} catch {
continue;
} // Legacy malformed snapshots cannot establish player records.
recordLeaderboardTelemetry(handle, report);
}
handle.prepare("INSERT INTO settings (key, value) VALUES ('leaderboard_v1', '1')").run();
src/lib/server/fleet/leaderboard-records.ts
+39 -12
View File
@@ -1,5 +1,10 @@
import type Database from 'better-sqlite3';
import type { LeaderboardPlayerCount, PersonalBestMetric, PersonalBestRecords, RecordEvidence } from '../../leaderboard.js';
import type {
LeaderboardPlayerCount,
PersonalBestMetric,
PersonalBestRecords,
RecordEvidence
} from '../../leaderboard.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
interface RecordRequest extends Record<PersonalBestMetric, number> {
@@ -10,14 +15,27 @@ interface RecordRequest extends Record<PersonalBestMetric, number> {
}
/** Fetch evidence only for requested players and scopes, never rank the whole fleet's history. */
export function getPersonalBestRecords(handle: Database.Database, requests: RecordRequest[]): PersonalBestRecords[] {
export function getPersonalBestRecords(
handle: Database.Database,
requests: RecordRequest[]
): PersonalBestRecords[] {
const result: PersonalBestRecords[] = [];
// Page requests are at most 100 players; chunk unusually large profile map collections too.
for (let offset = 0; offset < requests.length; offset += 100) {
const batch = requests.slice(offset, offset + 100);
const parameters = batch.flatMap((request, index) => [index, request.guid, request.game ?? null,
request.map ?? null, request.players ?? null, request.highestRound, request.bestKills, request.bestScore]);
const rows = handle.prepare(`WITH requested (requestId, guid, game, map, players, highestRound, bestKills, bestScore) AS (
const parameters = batch.flatMap((request, index) => [
index,
request.guid,
request.game ?? null,
request.map ?? null,
request.players ?? null,
request.highestRound,
request.bestKills,
request.bestScore
]);
const rows = handle
.prepare(
`WITH requested (requestId, guid, game, map, players, highestRound, bestKills, bestScore) AS (
VALUES ${batch.map(() => '(?, ?, ?, ?, ?, ?, ?, ?)').join(', ')}
), candidates AS (
SELECT r.requestId, m.id AS matchId, m.game, m.map_key AS mapKey, m.map_name AS mapName,
@@ -41,14 +59,23 @@ export function getPersonalBestRecords(handle: Database.Database, requests: Reco
SELECT *, ROW_NUMBER() OVER (PARTITION BY requestId, metric ORDER BY completionOrder, matchId) AS choice
FROM valuesByMetric
) SELECT requestId, metric, value, matchId, game, mapKey, mapName, server, startedAt, endedAt,
startObserved, firstObservedRound, presentAtStart FROM ranked WHERE choice = 1`)
.all(...parameters) as Array<Omit<RecordEvidence, 'startObserved' | 'presentAtStart'> & {
requestId: number; metric: PersonalBestMetric; startObserved: number; presentAtStart: number | null;
}>;
const records = batch.map(() => ({} as PersonalBestRecords));
startObserved, firstObservedRound, presentAtStart FROM ranked WHERE choice = 1`
)
.all(...parameters) as Array<
Omit<RecordEvidence, 'startObserved' | 'presentAtStart'> & {
requestId: number;
metric: PersonalBestMetric;
startObserved: number;
presentAtStart: number | null;
}
>;
const records = batch.map(() => ({}) as PersonalBestRecords);
for (const { requestId, metric, startObserved, presentAtStart, ...record } of rows) {
records[requestId][metric] = { ...record, startObserved: Boolean(startObserved),
presentAtStart: presentAtStart === null ? null : Boolean(presentAtStart) };
records[requestId][metric] = {
...record,
startObserved: Boolean(startObserved),
presentAtStart: presentAtStart === null ? null : Boolean(presentAtStart)
};
}
result.push(...records);
}
src/lib/server/fleet/leaderboard.ts
+166 -54
View File
@@ -2,19 +2,34 @@ import { db } from '../db/index.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
import { getPersonalBestRecords } from './leaderboard-records.js';
import { countsAsGamePlayed } from './leaderboard-participation.js';
import { EFFICIENCY_MIN_GAMES, EFFICIENCY_MIN_KILLS, EFFICIENCY_MIN_DOWNS,
isEfficiencyMetric } from '../../leaderboard-efficiency.js';
import { leaderboardMetrics, type LeaderboardEntry, type LeaderboardMap, type LeaderboardMetric,
type LeaderboardOptions, type LeaderboardResponse } from '../../leaderboard.js';
import {
EFFICIENCY_MIN_GAMES,
EFFICIENCY_MIN_KILLS,
EFFICIENCY_MIN_DOWNS,
isEfficiencyMetric
} from '../../leaderboard-efficiency.js';
import {
leaderboardMetrics,
type LeaderboardEntry,
type LeaderboardMap,
type LeaderboardMetric,
type LeaderboardOptions,
type LeaderboardResponse
} from '../../leaderboard.js';
const metrics = new Set<string>(leaderboardMetrics.map(metric => metric.key));
const metrics = new Set<string>(leaderboardMetrics.map((metric) => metric.key));
export const MAX_LEADERBOARD_OFFSET = 1_000_000;
type RankedEntry = Omit<LeaderboardEntry, 'records'>;
interface RankedScope { revision: number; entries: RankedEntry[]; maps: LeaderboardMap[]; summary: LeaderboardResponse['summary'] }
interface RankedScope {
revision: number;
entries: RankedEntry[];
maps: LeaderboardMap[];
summary: LeaderboardResponse['summary'];
}
const scopeCaches = new WeakMap<object, Map<string, RankedScope>>();
// Match SQLite's built-in lower(), which folds ASCII rather than Unicode.
const searchText = (value: string) => value.replace(/[A-Z]/g, ch => ch.toLowerCase());
const searchText = (value: string) => value.replace(/[A-Z]/g, (ch) => ch.toLowerCase());
interface LeaderboardRow extends Omit<RankedEntry, 'guid'> {
// A left join keeps metadata available when the requested page has no players.
@@ -39,35 +54,78 @@ export function getLeaderboard(options: LeaderboardOptions = {}): LeaderboardRes
}
/** Select the exact player after ranking everyone eligible in the requested scope. */
export function getPlayerLeaderboardEntry(guid: string,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players' | 'metric'> = {}): LeaderboardEntry | null {
return readLeaderboard({ game: options.game, map: options.map, players: options.players, metric: options.metric, limit: 1 }, guid);
export function getPlayerLeaderboardEntry(
guid: string,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players' | 'metric'> = {}
): LeaderboardEntry | null {
return readLeaderboard(
{
game: options.game,
map: options.map,
players: options.players,
metric: options.metric,
limit: 1
},
guid
);
}
function addRecordSources(handle: ReturnType<typeof db>, entries: RankedEntry[], options: LeaderboardOptions): LeaderboardEntry[] {
const records = getPersonalBestRecords(handle, entries.map(entry => ({
...entry, game: options.game || undefined, map: options.map || undefined, players: options.players
})));
function addRecordSources(
handle: ReturnType<typeof db>,
entries: RankedEntry[],
options: LeaderboardOptions
): LeaderboardEntry[] {
const records = getPersonalBestRecords(
handle,
entries.map((entry) => ({
...entry,
game: options.game || undefined,
map: options.map || undefined,
players: options.players
}))
);
return entries.map((entry, index) => ({ ...entry, records: records[index] }));
}
function readLeaderboard(options: LeaderboardOptions): LeaderboardResponse;
function readLeaderboard(options: LeaderboardOptions, exactGuid: string): LeaderboardEntry | null;
function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): LeaderboardResponse | LeaderboardEntry | null {
function readLeaderboard(
options: LeaderboardOptions,
exactGuid?: string
): LeaderboardResponse | LeaderboardEntry | null {
const handle = db();
const metric = options.metric && isLeaderboardMetric(options.metric) ? options.metric : 'highestRound';
const limit = Number.isSafeInteger(options.limit) ? Math.min(100, Math.max(1, options.limit!)) : 50;
const offset = Number.isSafeInteger(options.offset) ? Math.min(MAX_LEADERBOARD_OFFSET, Math.max(0, options.offset!)) : 0;
const conditions = ["m.game IN ('t5', 't6')", "m.status = 'completed'", 'mp.has_stats = 1', `m.highest_round >= ${MIN_RECORDED_ROUND}`,
`mp.highest_round >= ${MIN_RECORDED_ROUND}`];
const metric =
options.metric && isLeaderboardMetric(options.metric) ? options.metric : 'highestRound';
const limit = Number.isSafeInteger(options.limit)
? Math.min(100, Math.max(1, options.limit!))
: 50;
const offset = Number.isSafeInteger(options.offset)
? Math.min(MAX_LEADERBOARD_OFFSET, Math.max(0, options.offset!))
: 0;
const conditions = [
"m.game IN ('t5', 't6')",
"m.status = 'completed'",
'mp.has_stats = 1',
`m.highest_round >= ${MIN_RECORDED_ROUND}`,
`mp.highest_round >= ${MIN_RECORDED_ROUND}`
];
const parameters: Array<string | number> = [];
if (options.game) { conditions.push('m.game = ?'); parameters.push(options.game); }
if (options.map) { conditions.push('m.map_key = ?'); parameters.push(options.map); }
if (options.players) { conditions.push('m.start_players = ?'); parameters.push(options.players); }
if (options.game) {
conditions.push('m.game = ?');
parameters.push(options.game);
}
if (options.map) {
conditions.push('m.map_key = ?');
parameters.push(options.map);
}
if (options.players) {
conditions.push('m.start_players = ?');
parameters.push(options.players);
}
const qualification = isEfficiencyMetric(metric)
? `WHERE measures.gamesPlayed >= ${EFFICIENCY_MIN_GAMES} AND measures.${metric} IS NOT NULL`
+ (metric === 'headshotRate' ? ` AND measures.kills >= ${EFFICIENCY_MIN_KILLS}` : '')
+ (metric === 'killsPerDown' ? ` AND measures.downs >= ${EFFICIENCY_MIN_DOWNS}` : '')
? `WHERE measures.gamesPlayed >= ${EFFICIENCY_MIN_GAMES} AND measures.${metric} IS NOT NULL` +
(metric === 'headshotRate' ? ` AND measures.kills >= ${EFFICIENCY_MIN_KILLS}` : '') +
(metric === 'killsPerDown' ? ` AND measures.downs >= ${EFFICIENCY_MIN_DOWNS}` : '')
: '';
// A board reuses observations for player totals and distinct match metadata.
// Exact-player requests only consume the totals, so they can stream the join once.
@@ -100,14 +158,31 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
FROM measures JOIN identities p ON p.guid = measures.guid AND p.recency = 1 ${qualification}
)`;
let revision = -1;
try { revision = (handle.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'").get() as { revision: number }).revision; }
catch { /* Standalone history tools can operate without cache metadata. */ }
try {
revision = (
handle
.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'")
.get() as { revision: number }
).revision;
} catch {
/* Standalone history tools can operate without cache metadata. */
}
let cache = scopeCaches.get(handle);
if (!cache) { cache = new Map(); scopeCaches.set(handle, cache); }
const cacheKey = JSON.stringify([options.game ?? '', options.map ?? '', options.players ?? '', metric]);
if (!cache) {
cache = new Map();
scopeCaches.set(handle, cache);
}
const cacheKey = JSON.stringify([
options.game ?? '',
options.map ?? '',
options.players ?? '',
metric
]);
let saved = cache.get(cacheKey);
if (!saved || revision < 0 || saved.revision !== revision) {
const rows = handle.prepare(`${aggregated}, playerSummary AS (
const rows = handle
.prepare(
`${aggregated}, playerSummary AS (
SELECT COUNT(*) AS players, COALESCE(SUM(kills), 0) AS kills,
COALESCE(SUM(roundsSurvived), 0) AS roundsSurvived,
COALESCE(MAX(highestRound), 0) AS highestRound FROM totals
@@ -121,36 +196,73 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
metadata.roundsSurvived AS summaryRounds, metadata.highestRound AS summaryHighestRound,
metadata.firstTrackedAt AS summaryFirstTrackedAt, metadata.updatedAt AS summaryUpdatedAt
FROM metadata LEFT JOIN ranked ON 1
ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`)
.all(...parameters) as LeaderboardRow[];
const rankedEntries = rows.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
.map(({ matchingTotal, summaryPlayers, summaryMatches, summaryKills, summaryRounds,
summaryHighestRound, summaryFirstTrackedAt, summaryUpdatedAt, ...entry }) => entry);
const metadata = rows[0];
const summary: LeaderboardResponse['summary'] = {
players: metadata.summaryPlayers, matches: metadata.summaryMatches, kills: metadata.summaryKills,
roundsSurvived: metadata.summaryRounds, highestRound: metadata.summaryHighestRound,
firstTrackedAt: metadata.summaryFirstTrackedAt, updatedAt: metadata.summaryUpdatedAt
};
// Keep qualifying map variants available, even after their server is removed.
const maps = handle.prepare(`SELECT map_key AS key, game, map, map_name AS name
ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`
)
.all(...parameters) as LeaderboardRow[];
const rankedEntries = rows
.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
.map(
({
matchingTotal,
summaryPlayers,
summaryMatches,
summaryKills,
summaryRounds,
summaryHighestRound,
summaryFirstTrackedAt,
summaryUpdatedAt,
...entry
}) => entry
);
const metadata = rows[0];
const summary: LeaderboardResponse['summary'] = {
players: metadata.summaryPlayers,
matches: metadata.summaryMatches,
kills: metadata.summaryKills,
roundsSurvived: metadata.summaryRounds,
highestRound: metadata.summaryHighestRound,
firstTrackedAt: metadata.summaryFirstTrackedAt,
updatedAt: metadata.summaryUpdatedAt
};
// Keep qualifying map variants available, even after their server is removed.
const maps = handle
.prepare(
`SELECT map_key AS key, game, map, map_name AS name
FROM leaderboard_matches WHERE game IN ('t5', 't6') AND status = 'completed'
AND highest_round >= ${MIN_RECORDED_ROUND}${options.game ? ' AND game = ?' : ''}
GROUP BY game, map_key HAVING id = MAX(id)
ORDER BY game, name COLLATE NOCASE, map_key`)
.all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
saved = { revision, entries: rankedEntries, maps, summary };
cache.delete(cacheKey);
if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
// Bound total retained players as well as the number of scopes.
while (cache.size > 8 || [...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000) cache.delete(cache.keys().next().value!);
ORDER BY game, name COLLATE NOCASE, map_key`
)
.all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
saved = { revision, entries: rankedEntries, maps, summary };
cache.delete(cacheKey);
if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
// Bound total retained players as well as the number of scopes.
while (
cache.size > 8 ||
[...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000
)
cache.delete(cache.keys().next().value!);
}
if (exactGuid !== undefined) {
const entry = saved.entries.find(entry => entry.guid === exactGuid);
const entry = saved.entries.find((entry) => entry.guid === exactGuid);
return entry ? addRecordSources(handle, [entry], options)[0] : null;
}
const query = searchText(options.q?.trim() ?? '');
const matching = query ? saved.entries.filter(entry => searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)) : saved.entries;
const matching = query
? saved.entries.filter(
(entry) =>
searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)
)
: saved.entries;
const entries = addRecordSources(handle, matching.slice(offset, offset + limit), options);
return { entries, maps: saved.maps, total: matching.length, limit, offset, metric, summary: saved.summary };
return {
entries,
maps: saved.maps,
total: matching.length,
limit,
offset,
metric,
summary: saved.summary
};
}
src/lib/server/fleet/library-metadata.ts
+64 -17
View File
@@ -6,7 +6,9 @@ export function scriptCatalog(kind?: ArtifactKind) {
const version = (alias: string) => `CASE WHEN ${alias}.id IS NOT NULL THEN json_object(
'id',${alias}.id,'version',${alias}.version,'release_version',${alias}.release_version,
'sha256',${alias}.sha256,'size',${alias}.size,'state',${alias}.state,'uploaded_at',${alias}.uploaded_at) END`;
const rows = db().prepare(`SELECT s.*, ${version('c')} current_json, ${version('v')} latest_json,
const rows = db()
.prepare(
`SELECT s.*, ${version('c')} current_json, ${version('v')} latest_json,
COALESCE(counts.n,0) versionCount, active.beta_id runningBetaId
FROM scripts s
LEFT JOIN (SELECT script_id, COUNT(*) n, MAX(version) latest FROM script_versions GROUP BY script_id) counts ON counts.script_id=s.id
@@ -14,32 +16,77 @@ export function scriptCatalog(kind?: ArtifactKind) {
LEFT JOIN script_versions c ON c.id=s.current_version_id
LEFT JOIN (SELECT bs.script_id, MIN(b.id) beta_id FROM beta_scripts bs JOIN betas b ON b.id=bs.beta_id
WHERE b.state IN ('running','ready-promote') GROUP BY bs.script_id) active ON active.script_id=s.id
${kind ? 'WHERE s.kind=?' : ''} ORDER BY s.name`).all(...(kind ? [kind] : [])) as Array<ScriptRow & {
current_json: string | null; latest_json: string | null; versionCount: number; runningBetaId: number | null;
}>;
const servers = db().prepare('SELECT name, game, map FROM servers ORDER BY name').all() as { name: string; game: string | null; map: string | null }[];
return rows.map(({ current_json, latest_json, ...row }) => ({ ...row,
currentVersion: current_json ? JSON.parse(current_json) as ScriptVersionRow : null,
latestVersion: latest_json ? JSON.parse(latest_json) as ScriptVersionRow : null,
servers: row.scope_type && row.scope_value ? servers.filter(server => server[row.scope_type!] === row.scope_value).map(server => server.name) : []
${kind ? 'WHERE s.kind=?' : ''} ORDER BY s.name`
)
.all(...(kind ? [kind] : [])) as Array<
ScriptRow & {
current_json: string | null;
latest_json: string | null;
versionCount: number;
runningBetaId: number | null;
}
>;
const servers = db().prepare('SELECT name, game, map FROM servers ORDER BY name').all() as {
name: string;
game: string | null;
map: string | null;
}[];
return rows.map(({ current_json, latest_json, ...row }) => ({
...row,
currentVersion: current_json ? (JSON.parse(current_json) as ScriptVersionRow) : null,
latestVersion: latest_json ? (JSON.parse(latest_json) as ScriptVersionRow) : null,
servers:
row.scope_type && row.scope_value
? servers
.filter((server) => server[row.scope_type!] === row.scope_value)
.map((server) => server.name)
: []
}));
}
export function betaScriptMetadata(ids: number[]) {
const rows = db().prepare(`SELECT bs.beta_id, bs.script_id id, COALESCE(s.name,'(deleted)') name,
const rows = db()
.prepare(
`SELECT bs.beta_id, bs.script_id id, COALESCE(s.name,'(deleted)') name,
COALESCE(s.target_rel_path,'') target_rel_path, COALESCE(s.kind,bs.kind,'script') kind,
v.id version_id, v.version, v.release_version, v.sha256
FROM beta_scripts bs LEFT JOIN scripts s ON s.id=bs.script_id
LEFT JOIN script_versions v ON v.id=bs.script_version_id
WHERE bs.beta_id IN (SELECT value FROM json_each(?)) ORDER BY bs.script_id`).all(JSON.stringify(ids)) as Array<{
beta_id: number; id: number; name: string; target_rel_path: string; kind: ArtifactKind;
version_id: number | null; version: number; release_version: string | null; sha256: string;
}>;
const grouped = new Map<number, Array<{ id: number; name: string; target_rel_path: string; kind: ArtifactKind;
version: { id: number; version: number; release_version: string | null; sha256: string } | null }>>();
WHERE bs.beta_id IN (SELECT value FROM json_each(?)) ORDER BY bs.script_id`
)
.all(JSON.stringify(ids)) as Array<{
beta_id: number;
id: number;
name: string;
target_rel_path: string;
kind: ArtifactKind;
version_id: number | null;
version: number;
release_version: string | null;
sha256: string;
}>;
const grouped = new Map<
number,
Array<{
id: number;
name: string;
target_rel_path: string;
kind: ArtifactKind;
version: {
id: number;
version: number;
release_version: string | null;
sha256: string;
} | null;
}>
>();
for (const { beta_id, version_id, version, release_version, sha256, ...script } of rows) {
const group = grouped.get(beta_id) ?? [];
group.push({ ...script, version: version_id === null ? null : { id: version_id, version, release_version, sha256 } });
group.push({
...script,
version:
version_id === null ? null : { id: version_id, version, release_version, sha256 }
});
grouped.set(beta_id, group);
}
return grouped;
src/lib/server/fleet/lifecycle-evidence.ts
+15 -6
View File
@@ -2,18 +2,27 @@ import { db } from '../db/index.js';
/** Reports from before the latest lifecycle submission describe the old process. */
export function latestGameSubmission(server: string): string | null {
return (db().prepare(`SELECT MAX(at) AS at FROM (
return (
db()
.prepare(
`SELECT MAX(at) AS at FROM (
SELECT MAX(submitted_at) AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`
)
.get(server, server) as { at: string | null }
).at;
}
/** Look up each registered server through the covering submission index. */
export function latestGameSubmissions(): Map<string, string | null> {
const rows = db().prepare(`SELECT server, MAX(at) AS at FROM (
const rows = db()
.prepare(
`SELECT server, MAX(at) AS at FROM (
SELECT s.name AS server, (SELECT MAX(submitted_at) FROM server_deployments
WHERE server=s.name AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')) AS at
FROM servers s
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as
{ server: string; at: string | null }[];
return new Map(rows.map(row => [row.server, row.at]));
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`
)
.all() as { server: string; at: string | null }[];
return new Map(rows.map((row) => [row.server, row.at]));
}
src/lib/server/fleet/logs.ts
+60 -15
View File
@@ -14,7 +14,8 @@ export const KNOWN_LOGS = {
} as const;
export type LogKey = keyof typeof KNOWN_LOGS;
export const gameLogKey = (server: string): LogKey => gameOfServer(server) === 't6' ? 'games_zm' : 'games_sp';
export const gameLogKey = (server: string): LogKey =>
gameOfServer(server) === 't6' ? 'games_zm' : 'games_sp';
export function isLogKey(v: string): v is LogKey {
return Object.prototype.hasOwnProperty.call(KNOWN_LOGS, v);
@@ -45,33 +46,62 @@ export function resolveLogPath(serverName: string, key: LogKey): string | null {
export interface TailResult {
lines: string[];
size: number;
mtime: string; // ISO
mtime: string; // ISO
truncated: boolean; // true when we didn't read from the start of the file
}
export interface LogCursor {
fileId: string; offset: number; size: number; modified: string;
anchorLength: number; anchorHash: string;
fileId: string;
offset: number;
size: number;
modified: string;
anchorLength: number;
anchorHash: string;
}
export interface LogChunk {
contentBase64: string; size: number; mtime: string; truncated: boolean;
cursor: LogCursor; reset: boolean;
contentBase64: string;
size: number;
mtime: string;
truncated: boolean;
cursor: LogCursor;
reset: boolean;
}
/** Bounded byte reads; an anchor before the offset detects copy-truncate and regrowth. */
export async function readLogChunk(path: string, cursor?: LogCursor): Promise<LogChunk | TailResult | null> {
export async function readLogChunk(
path: string,
cursor?: LogCursor
): Promise<LogChunk | TailResult | null> {
const remote = await nativeLogChunk(path, cursor);
if (remote !== undefined) return remote;
let file;
try { file = await open(path, 'r'); }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; throw error; }
try {
file = await open(path, 'r');
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw error;
}
try {
const st = await file.stat();
if (!st.isFile()) return null;
const fileId = `${st.dev}:${st.ino}`, modified = `${st.mtimeMs}:${st.ctimeMs}`, size = st.size;
const fileId = `${st.dev}:${st.ino}`,
modified = `${st.mtimeMs}:${st.ctimeMs}`,
size = st.size;
const mtime = st.mtime.toISOString();
if (cursor?.fileId === fileId && cursor.modified === modified && cursor.size === size && cursor.offset === size) {
return { contentBase64: '', size, mtime, truncated: cursor.offset > 0, cursor, reset: false };
if (
cursor?.fileId === fileId &&
cursor.modified === modified &&
cursor.size === size &&
cursor.offset === size
) {
return {
contentBase64: '',
size,
mtime,
truncated: cursor.offset > 0,
cursor,
reset: false
};
}
const hash = (value: Buffer) => createHash('sha256').update(value).digest('hex');
let reset = !cursor || cursor.fileId !== fileId || cursor.offset > size;
@@ -86,9 +116,24 @@ export async function readLogChunk(path: string, cursor?: LogCursor): Promise<Lo
const offset = start + read.bytesRead;
const anchor = Buffer.alloc(Math.min(256, offset));
await file.read(anchor, 0, anchor.length, offset - anchor.length);
return { contentBase64: content.subarray(0, read.bytesRead).toString('base64'), size, mtime, truncated: start > 0, reset,
cursor: { fileId, modified, size, offset, anchorLength: anchor.length, anchorHash: hash(anchor) } };
} finally { await file.close(); }
return {
contentBase64: content.subarray(0, read.bytesRead).toString('base64'),
size,
mtime,
truncated: start > 0,
reset,
cursor: {
fileId,
modified,
size,
offset,
anchorLength: anchor.length,
anchorHash: hash(anchor)
}
};
} finally {
await file.close();
}
}
const DEFAULT_TAIL_LINES = 200;
src/lib/server/fleet/match-history.ts
+92 -27
View File
@@ -1,6 +1,12 @@
import { db } from '../db/index.js';
import { matchChatTimeline } from './chat.js';
import { MIN_RECORDED_ROUND, type MatchDetail, type MatchPlayer, type MatchHistoryResponse, type ServerMatch } from '../../match-history.js';
import {
MIN_RECORDED_ROUND,
type MatchDetail,
type MatchPlayer,
type MatchHistoryResponse,
type ServerMatch
} from '../../match-history.js';
import { readMatchQuery, type MatchHistoryQuery } from '../../match-history-query.js';
@@ -15,29 +21,69 @@ const matchSummary = `m.id, m.game, m.highest_round AS observedRound, m.server,
MAX(0, ROUND((julianday(m.ended_at) - julianday(m.started_at)) * 86400000)) END AS durationMs,
CASE WHEN m.status = 'completed' THEN m.highest_round END AS finalRound`;
export function serverMatchHistory(server?: string, limit = 25, query: MatchHistoryQuery = readMatchQuery(new URLSearchParams()), now = Date.now()): MatchHistoryResponse {
export function serverMatchHistory(
server?: string,
limit = 25,
query: MatchHistoryQuery = readMatchQuery(new URLSearchParams()),
now = Date.now()
): MatchHistoryResponse {
const handle = db();
const clauses = [`m.highest_round >= ${MIN_RECORDED_ROUND}`];
const parameters: Array<string | number> = [];
if (server !== undefined) { clauses.push('m.server = ?'); parameters.push(server); }
if (query.game) { clauses.push('m.game = ?'); parameters.push(query.game); }
const maps = handle.prepare(`SELECT m.map_key AS key, MAX(m.map_name) AS name, m.game
FROM leaderboard_matches m WHERE ${clauses.join(' AND ')} GROUP BY m.map_key, m.game ORDER BY name COLLATE NOCASE, game`)
if (server !== undefined) {
clauses.push('m.server = ?');
parameters.push(server);
}
if (query.game) {
clauses.push('m.game = ?');
parameters.push(query.game);
}
const maps = handle
.prepare(
`SELECT m.map_key AS key, MAX(m.map_name) AS name, m.game
FROM leaderboard_matches m WHERE ${clauses.join(' AND ')} GROUP BY m.map_key, m.game ORDER BY name COLLATE NOCASE, game`
)
.all(...parameters) as MatchHistoryResponse['maps'];
if (query.map) { clauses.push('m.map_key = ?'); parameters.push(query.map); }
if (query.status) { clauses.push('m.status = ?'); parameters.push(query.status); }
if (query.days) { clauses.push('julianday(m.started_at) >= julianday(?)'); parameters.push(new Date(now - query.days * 86400000).toISOString()); }
if (query.map) {
clauses.push('m.map_key = ?');
parameters.push(query.map);
}
if (query.status) {
clauses.push('m.status = ?');
parameters.push(query.status);
}
if (query.days) {
clauses.push('julianday(m.started_at) >= julianday(?)');
parameters.push(new Date(now - query.days * 86400000).toISOString());
}
// Search readable instance names as well as their stored identifiers. Tokens allow "Town Lobby 1".
for (const token of query.q.toLowerCase().replace(/\blobby\s+(?=\d+\b)/g, '').split(/[\s_]+/).filter(Boolean)) {
clauses.push("instr(lower(m.server || ' ' || m.map_name), ?) > 0"); parameters.push(token);
for (const token of query.q
.toLowerCase()
.replace(/\blobby\s+(?=\d+\b)/g, '')
.split(/[\s_]+/)
.filter(Boolean)) {
clauses.push("instr(lower(m.server || ' ' || m.map_name), ?) > 0");
parameters.push(token);
}
const where = `WHERE ${clauses.join(' AND ')}`;
const total = (handle.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`).get(...parameters) as { total: number }).total;
const total = (
handle
.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`)
.get(...parameters) as { total: number }
).total;
const page = Math.min(query.page, Math.max(1, Math.ceil(total / limit)));
const column = { started: 'startedAt', round: 'observedRound', duration: 'durationMs', players: 'uniquePlayers' }[query.sort];
const column = {
started: 'startedAt',
round: 'observedRound',
duration: 'durationMs',
players: 'uniquePlayers'
}[query.sort];
const direction = query.direction === 'asc' ? 'ASC' : 'DESC';
const matches = handle.prepare(`SELECT ${matchSummary} FROM leaderboard_matches m
${where} ORDER BY ${column} IS NULL ASC, ${column} ${direction}, m.id DESC LIMIT ? OFFSET ?`)
const matches = handle
.prepare(
`SELECT ${matchSummary} FROM leaderboard_matches m
${where} ORDER BY ${column} IS NULL ASC, ${column} ${direction}, m.id DESC LIMIT ? OFFSET ?`
)
.all(...parameters, limit, (page - 1) * limit) as ServerMatch[];
return { matches, total, page, limit, maps };
}
@@ -46,13 +92,20 @@ export function serverMatchHistory(server?: string, limit = 25, query: MatchHist
export function getMatchDetail(id: number): MatchDetail | null {
if (!Number.isSafeInteger(id) || id < 1) return null;
const handle = db();
const match = handle.prepare(`SELECT ${matchSummary}, m.start_observed AS startObserved
const match = handle
.prepare(
`SELECT ${matchSummary}, m.start_observed AS startObserved
FROM leaderboard_matches m WHERE m.id = ? AND m.status = 'completed'
AND m.highest_round >= ${MIN_RECORDED_ROUND}`)
.get(id) as (Omit<MatchDetail, 'players' | 'chat' | 'startObserved'> & { startObserved: number }) | undefined;
AND m.highest_round >= ${MIN_RECORDED_ROUND}`
)
.get(id) as
| (Omit<MatchDetail, 'players' | 'chat' | 'startObserved'> & { startObserved: number })
| undefined;
if (!match) return null;
const players = handle.prepare(`SELECT p.guid,
const players = handle
.prepare(
`SELECT p.guid,
COALESCE((SELECT identity.name FROM leaderboard_players identity
WHERE identity.guid = p.guid AND identity.game IN ('t5', 't6')
ORDER BY identity.last_seen_at DESC, identity.game LIMIT 1),
@@ -76,12 +129,24 @@ export function getMatchDetail(id: number): MatchDetail | null {
AND history.highest_round >= ${MIN_RECORDED_ROUND}
AND recorded.highest_round >= ${MIN_RECORDED_ROUND})) AS canViewProfile
FROM leaderboard_match_players p WHERE p.match_id = ?
ORDER BY hasStats DESC, kills DESC, highestRound DESC, name COLLATE NOCASE, p.guid`)
.all(match.game, id) as Array<Omit<MatchPlayer, 'hasStats' | 'presentAtEnd' | 'canViewProfile'> & {
hasStats: number; presentAtEnd: number; canViewProfile: number;
}>;
return { ...match, chat: matchChatTimeline(id), startObserved: Boolean(match.startObserved), players: players.map(player => ({
...player, hasStats: Boolean(player.hasStats), presentAtEnd: Boolean(player.presentAtEnd),
canViewProfile: Boolean(player.canViewProfile)
})) };
ORDER BY hasStats DESC, kills DESC, highestRound DESC, name COLLATE NOCASE, p.guid`
)
.all(match.game, id) as Array<
Omit<MatchPlayer, 'hasStats' | 'presentAtEnd' | 'canViewProfile'> & {
hasStats: number;
presentAtEnd: number;
canViewProfile: number;
}
>;
return {
...match,
chat: matchChatTimeline(id),
startObserved: Boolean(match.startObserved),
players: players.map((player) => ({
...player,
hasStats: Boolean(player.hasStats),
presentAtEnd: Boolean(player.presentAtEnd),
canViewProfile: Boolean(player.canViewProfile)
}))
};
}
src/lib/server/fleet/metrics.ts
+196 -49
View File
@@ -1,8 +1,18 @@
import { fleetSettings } from '../settings.js';
import { db } from '../db/index.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { parseMetricsQuery, type HeatmapCell, type MapMetrics, type MetricPoint, type MetricsData,
type MetricsDays, type MetricsQuery, type PreviousMetrics, type ServerMetrics, type Utilization } from '../../metrics.js';
import {
parseMetricsQuery,
type HeatmapCell,
type MapMetrics,
type MetricPoint,
type MetricsData,
type MetricsDays,
type MetricsQuery,
type PreviousMetrics,
type ServerMetrics,
type Utilization
} from '../../metrics.js';
export type { MapMetrics, Utilization } from '../../metrics.js';
@@ -17,10 +27,13 @@ export interface MetricSample {
export function recordMetrics(samples: MetricSample[], at = Date.now()): void {
const handle = db();
handle.transaction(() => {
const insert = handle.prepare('INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)');
const insert = handle.prepare(
'INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)'
);
for (const s of samples) insert.run(at, s.server, s.game, s.map, s.players, s.capacity);
const days = fleetSettings().metricsKeepDays;
if (days) handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - days * 86400_000);
if (days)
handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - days * 86400_000);
})();
}
@@ -42,52 +55,120 @@ function addObservation(total: Utilization, players: number | null, hours: numbe
* capped at the freshness limit; longer gaps contribute no observed or unknown time.
* Read the indexed history once and share the intervals across every aggregation. */
export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()): MetricsData {
const query = parseMetricsQuery(new URLSearchParams(typeof input === 'number' ? { days: String(input) }
: { days: String(input.days), game: input.game ?? '', map: input.map ?? '', timeZone: input.timeZone }));
const query = parseMetricsQuery(
new URLSearchParams(
typeof input === 'number'
? { days: String(input) }
: {
days: String(input.days),
game: input.game ?? '',
map: input.map ?? '',
timeZone: input.timeZone
}
)
);
const { days, ...scope } = query;
const since = now - days * 86400_000;
const bucketMs = days === 1 ? 3600_000 : 86400_000;
const handle = db();
// These small grouped results keep options and retained evidence independent
// of the chosen scope/window without transferring a second raw history scan.
const groups = handle.prepare(`SELECT game, map, MIN(at) AS first, MAX(at) AS latest
FROM metric_samples WHERE at <= ? GROUP BY game, map ORDER BY game, map`).all(now) as
{ game: string; map: string; first: number; latest: number }[];
const groups = handle
.prepare(
`SELECT game, map, MIN(at) AS first, MAX(at) AS latest
FROM metric_samples WHERE at <= ? GROUP BY game, map ORDER BY game, map`
)
.all(now) as { game: string; map: string; first: number; latest: number }[];
const matchesScope = (sample: { game: string; map: string }) =>
(scope.game === null || sample.game === scope.game) && (scope.map === null || sample.map === scope.map);
const evidenceBounds = (values: typeof groups) => values.reduce<{ first: number | null; latest: number | null }>((bounds, group) => ({
first: bounds.first === null ? group.first : Math.min(bounds.first, group.first),
latest: bounds.latest === null ? group.latest : Math.max(bounds.latest, group.latest)
}), { first: null, latest: null });
(scope.game === null || sample.game === scope.game) &&
(scope.map === null || sample.map === scope.map);
const evidenceBounds = (values: typeof groups) =>
values.reduce<{ first: number | null; latest: number | null }>(
(bounds, group) => ({
first: bounds.first === null ? group.first : Math.min(bounds.first, group.first),
latest:
bounds.latest === null ? group.latest : Math.max(bounds.latest, group.latest)
}),
{ first: null, latest: null }
);
const globalBounds = evidenceBounds(groups);
const scopedBounds = evidenceBounds(groups.filter(matchesScope));
const registered = new Set((handle.prepare('SELECT name FROM servers WHERE destroyed_at IS NULL').all() as { name: string }[])
.map(server => server.name));
const metadata = { firstRetainedSampleAt: globalBounds.first, latestSampleAt: globalBounds.latest,
registeredServerCount: registered.size, scopeFirstSampleAt: scopedBounds.first, scopeLatestSampleAt: scopedBounds.latest };
const scopeOptions = { games: [...new Set(groups.map(group => group.game))], maps: groups.map(({ game, map }) => ({ game, map })) };
const registered = new Set(
(
handle.prepare('SELECT name FROM servers WHERE destroyed_at IS NULL').all() as {
name: string;
}[]
).map((server) => server.name)
);
const metadata = {
firstRetainedSampleAt: globalBounds.first,
latestSampleAt: globalBounds.latest,
registeredServerCount: registered.size,
scopeFirstSampleAt: scopedBounds.first,
scopeLatestSampleAt: scopedBounds.latest
};
const scopeOptions = {
games: [...new Set(groups.map((group) => group.game))],
maps: groups.map(({ game, map }) => ({ game, map }))
};
const summary = emptyUtilization();
const previous: PreviousMetrics = { since: since - days * 86400_000, until: since, summary: emptyUtilization(),
firstSampleAt: null, lastSampleAt: null, partialHistory: false };
const previous: PreviousMetrics = {
since: since - days * 86400_000,
until: since,
summary: emptyUtilization(),
firstSampleAt: null,
lastSampleAt: null,
partialHistory: false
};
const serverTotals = new Map<string, ServerMetrics>();
const mapTotals = new Map<string, Map<string, MapMetrics>>();
const points: MetricPoint[] = [];
for (let at = Math.floor(since / bucketMs) * bucketMs; at < now; at += bucketMs) {
const end = Math.min(at + bucketMs, now);
const start = Math.max(at, since);
points.push({ at, startAt: start, endAt: end, partial: start !== at || end !== at + bucketMs, ...emptyUtilization() });
points.push({
at,
startAt: start,
endAt: end,
partial: start !== at || end !== at + bucketMs,
...emptyUtilization()
});
}
const cells: HeatmapCell[] = Array.from({ length: 7 * 24 }, (_, index) => ({
weekday: Math.floor(index / 24), hour: index % 24, distinctDates: 0, ...emptyUtilization()
weekday: Math.floor(index / 24),
hour: index % 24,
distinctDates: 0,
...emptyUtilization()
}));
const cellDates = cells.map(() => new Set<string>());
const calendar = new Intl.DateTimeFormat('en-US', { timeZone: scope.timeZone,
year: 'numeric', month: '2-digit', day: '2-digit', weekday: 'short', hour: 'numeric', hourCycle: 'h23' });
const weekdays: Record<string, number> = { Mon: 0, Tue: 1, Wed: 2, Thu: 3, Fri: 4, Sat: 5, Sun: 6 };
const calendar = new Intl.DateTimeFormat('en-US', {
timeZone: scope.timeZone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
weekday: 'short',
hour: 'numeric',
hourCycle: 'h23'
});
const weekdays: Record<string, number> = {
Mon: 0,
Tue: 1,
Wed: 2,
Thu: 3,
Fri: 4,
Sat: 5,
Sun: 6
};
let cachedMinute = NaN;
let localCell = { index: 0, date: '' };
interface BatchCounts { known: number; active: number; idle: number; unknown: number; players: number }
interface BatchCounts {
known: number;
active: number;
idle: number;
unknown: number;
players: number;
}
function addHeatmap(counts: BatchCounts, start: number, end: number): void {
// Samples are bounded by the shared freshness window. Splitting at UTC minute boundaries
// handles local hour/day boundaries, DST, and fractional-hour zones.
@@ -95,8 +176,13 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
for (let cursor = start; cursor < end;) {
const minute = Math.floor(cursor / 60_000);
if (minute !== cachedMinute) {
const parts = Object.fromEntries(calendar.formatToParts(cursor).map(part => [part.type, part.value]));
localCell = { index: weekdays[parts.weekday] * 24 + Number(parts.hour), date: `${parts.year}-${parts.month}-${parts.day}` };
const parts = Object.fromEntries(
calendar.formatToParts(cursor).map((part) => [part.type, part.value])
);
localCell = {
index: weekdays[parts.weekday] * 24 + Number(parts.hour),
date: `${parts.year}-${parts.month}-${parts.day}`
};
cachedMinute = minute;
}
const stop = Math.min(end, (minute + 1) * 60_000);
@@ -120,11 +206,13 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
const samples = scope.game === null ? batch : batch.filter(matchesScope);
if (!samples.length) return;
const end = Math.min(nextAt, at + ONLINE_MAX_MS, now);
const priorHours = Math.max(0, Math.min(end, previous.until) - Math.max(at, previous.since)) / 3600_000;
const priorHours =
Math.max(0, Math.min(end, previous.until) - Math.max(at, previous.since)) / 3600_000;
if (priorHours > 0) {
previous.firstSampleAt ??= at;
previous.lastSampleAt = at;
for (const sample of samples) addObservation(previous.summary, sample.players, priorHours);
for (const sample of samples)
addObservation(previous.summary, sample.players, priorHours);
}
// Preserve current-window zero-duration rows without bringing rows from
// the separate comparison window into the visible server/map tables.
@@ -146,14 +234,27 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
for (const sample of samples) {
let server = serverTotals.get(sample.server);
if (!server) {
server = { server: sample.server, currentRegistered: registered.has(sample.server), ...emptyUtilization() };
server = {
server: sample.server,
currentRegistered: registered.has(sample.server),
...emptyUtilization()
};
serverTotals.set(sample.server, server);
}
let game = mapTotals.get(sample.game);
if (!game) { game = new Map(); mapTotals.set(sample.game, game); }
if (!game) {
game = new Map();
mapTotals.set(sample.game, game);
}
let map = game.get(sample.map);
if (!map) {
map = { game: sample.game, map: sample.map, full: 0, capacityObserved: 0, ...emptyUtilization() };
map = {
game: sample.game,
map: sample.map,
full: 0,
capacityObserved: 0,
...emptyUtilization()
};
game.set(sample.map, map);
}
addObservation(summary, sample.players, hours);
@@ -167,10 +268,14 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
if (sample.players > 0) counts.active++;
else if (sample.players === 0) counts.idle++;
}
const known = sample.players !== null && sample.capacity !== null && sample.capacity > 0;
const known =
sample.players !== null && sample.capacity !== null && sample.capacity > 0;
const full = known && sample.players! >= sample.capacity!;
const previous = pressure.get(map);
pressure.set(map, { known: known && (previous?.known ?? true), full: full && (previous?.full ?? true) });
pressure.set(map, {
known: known && (previous?.known ?? true),
full: full && (previous?.full ?? true)
});
}
for (const [map, state] of pressure) {
if (state.known) {
@@ -181,9 +286,21 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
addHeatmap(counts, start, end);
}
type SampleRow = [at: number, server: string, game: string, map: string, players: number | null, capacity: number | null];
const samples = handle.prepare(`SELECT at, server, game, map, players, capacity FROM metric_samples
WHERE at >= ? AND at <= ? ORDER BY at`).raw().iterate(previous.since - ONLINE_MAX_MS, now) as IterableIterator<SampleRow>;
type SampleRow = [
at: number,
server: string,
game: string,
map: string,
players: number | null,
capacity: number | null
];
const samples = handle
.prepare(
`SELECT at, server, game, map, players, capacity FROM metric_samples
WHERE at >= ? AND at <= ? ORDER BY at`
)
.raw()
.iterate(previous.since - ONLINE_MAX_MS, now) as IterableIterator<SampleRow>;
for (const [at, server, game, map, players, capacity] of samples) {
if (batch.length && at !== batch[0].at) {
aggregateBatch(at);
@@ -193,16 +310,46 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
}
aggregateBatch(now);
const compareText = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0;
const servers = [...serverTotals.values()].sort((a, b) => b.active - a.active || compareText(a.server, b.server));
const maps = [...mapTotals.values()].flatMap(game => [...game.values()])
.sort((a, b) => b.playerHours - a.playerHours || compareText(a.map, b.map) || compareText(a.game, b.game));
previous.partialHistory = scopedBounds.first === null || scopedBounds.first > previous.since || previous.firstSampleAt === null;
for (let index = 0; index < cells.length; index++) cells[index].distinctDates = cellDates[index].size;
const compareText = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);
const servers = [...serverTotals.values()].sort(
(a, b) => b.active - a.active || compareText(a.server, b.server)
);
const maps = [...mapTotals.values()]
.flatMap((game) => [...game.values()])
.sort(
(a, b) =>
b.playerHours - a.playerHours ||
compareText(a.map, b.map) ||
compareText(a.game, b.game)
);
previous.partialHistory =
scopedBounds.first === null ||
scopedBounds.first > previous.since ||
previous.firstSampleAt === null;
for (let index = 0; index < cells.length; index++)
cells[index].distinctDates = cellDates[index].size;
const fullMapHours = maps.reduce((total, map) => total + map.full, 0);
const knownMapHours = maps.reduce((total, map) => total + map.capacityObserved, 0);
const pooledPressure = { value: knownMapHours > 0 ? 100 * fullMapHours / knownMapHours : null,
fullMapHours, knownMapHours, coveredMaps: maps.filter(map => map.capacityObserved > 0).length, totalMaps: maps.length };
return { days, since, now, ...metadata, scope, scopeOptions, summary, servers, maps, points, previous,
heatmap: { timeZone: scope.timeZone, cells }, pooledPressure };
const pooledPressure = {
value: knownMapHours > 0 ? (100 * fullMapHours) / knownMapHours : null,
fullMapHours,
knownMapHours,
coveredMaps: maps.filter((map) => map.capacityObserved > 0).length,
totalMaps: maps.length
};
return {
days,
since,
now,
...metadata,
scope,
scopeOptions,
summary,
servers,
maps,
points,
previous,
heatmap: { timeZone: scope.timeZone, cells },
pooledPressure
};
}
src/lib/server/fleet/monitoring-snapshot.ts
+133 -38
View File
@@ -10,9 +10,22 @@ import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { serverObservation } from './status.js';
import { recoverySettling } from '../provisioning/restarts.js';
const nativeFields = ['id','server','host_id','generation','library_id','image','bind_address','port','public_address','public_port','created_at','retired_at'];
const nativeFields = [
'id',
'server',
'host_id',
'generation',
'library_id',
'image',
'bind_address',
'port',
'public_address',
'public_port',
'created_at',
'retired_at'
];
const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS instance_generation,n.library_id AS game_library_id,
CASE WHEN n.id IS NOT NULL THEN json_object(${nativeFields.map(key => `'${key}',n.${key}`).join(',')}) END AS instance_json,
CASE WHEN n.id IS NOT NULL THEN json_object(${nativeFields.map((key) => `'${key}',n.${key}`).join(',')}) END AS instance_json,
h.inventory AS host_inventory,h.last_heartbeat AS host_heartbeat,
EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id=h.id AND c.revoked_at IS NULL) AS credential_active,
(SELECT json_object('id',d.id,'action',d.action,'state',d.state,'submitted_at',d.submitted_at)
@@ -27,65 +40,147 @@ const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS in
LEFT JOIN ec_hosts h ON h.id=n.host_id AND h.deleted_at IS NULL
LEFT JOIN restart_rollout r ON r.server=s.name LEFT JOIN server_telemetry t ON t.server=s.name`;
interface Row extends ServerRow {
instance_json: string | null; host_inventory: string | null; host_heartbeat: string | null; credential_active: number;
lifecycle_json: string | null; lifecycle_at: string | null; submission: string | null;
restart_at: string | null; restart_healthy: string | null; restart_operation_id: string | null; telemetry_at: string | null; telemetry_json: string | null;
instance_json: string | null;
host_inventory: string | null;
host_heartbeat: string | null;
credential_active: number;
lifecycle_json: string | null;
lifecycle_at: string | null;
submission: string | null;
restart_at: string | null;
restart_healthy: string | null;
restart_operation_id: string | null;
telemetry_at: string | null;
telemetry_json: string | null;
}
interface Record {
server: ServerRow; native: NativeInstance | null;
server: ServerRow;
native: NativeInstance | null;
host: Pick<Host, 'connected' | 'connectionState' | 'inventory'> | null;
lifecycle: { id: string; action: string; state: string; submitted_at: string | null } | null;
lifecycleAt: string | null; submission: string | null; telemetry: GameTelemetry | null;
restartAt: string | null; restartHealthy: string | null; restartOperationId: string | null;
lifecycleAt: string | null;
submission: string | null;
telemetry: GameTelemetry | null;
restartAt: string | null;
restartHealthy: string | null;
restartOperationId: string | null;
}
/** A pass shares its reads; each asynchronous evidence boundary refreshes in one joined query. */
export class MonitoringSnapshot {
private records = new Map<string, Record>();
constructor(serverName?: string) { ensureHostSchema(db()); this.refresh(serverName); }
constructor(serverName?: string) {
ensureHostSchema(db());
this.refresh(serverName);
}
refresh(server?: string): void {
const rows = server === undefined ? db().prepare(sql + ' WHERE s.destroyed_at IS NULL ORDER BY s.name').all() as Row[]
: db().prepare(sql + ' WHERE s.name=? AND s.destroyed_at IS NULL').all(server) as Row[];
if (server === undefined) this.records.clear(); else this.records.delete(server);
const rows =
server === undefined
? (db()
.prepare(sql + ' WHERE s.destroyed_at IS NULL ORDER BY s.name')
.all() as Row[])
: (db()
.prepare(sql + ' WHERE s.name=? AND s.destroyed_at IS NULL')
.all(server) as Row[]);
if (server === undefined) this.records.clear();
else this.records.delete(server);
for (const row of rows) {
const { instance_json,host_inventory,host_heartbeat,credential_active,lifecycle_json,lifecycle_at,submission,
restart_at,restart_healthy,restart_operation_id,telemetry_at,telemetry_json,...fields } = row;
const connected = !!credential_active && !!host_heartbeat && Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
const {
instance_json,
host_inventory,
host_heartbeat,
credential_active,
lifecycle_json,
lifecycle_at,
submission,
restart_at,
restart_healthy,
restart_operation_id,
telemetry_at,
telemetry_json,
...fields
} = row;
const connected =
!!credential_active &&
!!host_heartbeat &&
Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
let telemetry: GameTelemetry | null = null;
if (telemetry_json) try {
const saved = JSON.parse(telemetry_json) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.name) telemetry = { ...event, receivedAt: telemetry_at!, timing: saved.timing };
} catch { /* Invalid telemetry cannot establish health. */ }
this.records.set(row.name, { server: fields, native: instance_json ? JSON.parse(instance_json) : null,
host: host_inventory ? { connected, connectionState: connected ? 'online' : 'offline',
inventory: parseHostInventory(row.host_id!, host_inventory) } : null,
lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null, lifecycleAt: lifecycle_at, submission, telemetry,
restartAt: restart_at, restartHealthy: restart_healthy, restartOperationId: restart_operation_id });
if (telemetry_json)
try {
const saved = JSON.parse(telemetry_json) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.name)
telemetry = { ...event, receivedAt: telemetry_at!, timing: saved.timing };
} catch {
/* Invalid telemetry cannot establish health. */
}
this.records.set(row.name, {
server: fields,
native: instance_json ? JSON.parse(instance_json) : null,
host: host_inventory
? {
connected,
connectionState: connected ? 'online' : 'offline',
inventory: parseHostInventory(row.host_id!, host_inventory)
}
: null,
lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null,
lifecycleAt: lifecycle_at,
submission,
telemetry,
restartAt: restart_at,
restartHealthy: restart_healthy,
restartOperationId: restart_operation_id
});
}
}
get(server: string) { return this.records.get(server); }
servers() { return [...this.records.values()].map(record => record.server); }
get(server: string) {
return this.records.get(server);
}
servers() {
return [...this.records.values()].map((record) => record.server);
}
provisioning(server: string) {
const operation = this.get(server)?.lifecycle;
return !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state);
return (
!!operation &&
['create', 'rebuild', 'start'].includes(operation.action) &&
['queued', 'running', 'awaiting-health'].includes(operation.state)
);
}
settling(server: string) {
const record = this.get(server);
return recoverySettling(record?.restartAt ? { requested_at: record.restartAt,
healthy_at: record.restartHealthy, operation_id: record.restartOperationId } : undefined, record?.lifecycle);
return recoverySettling(
record?.restartAt
? {
requested_at: record.restartAt,
healthy_at: record.restartHealthy,
operation_id: record.restartOperationId
}
: undefined,
record?.lifecycle
);
}
telemetryEvidence(server: string) {
const record = this.get(server), report = record?.telemetry;
const record = this.get(server),
report = record?.telemetry;
if (!report) return null;
const received = Date.parse(report.receivedAt), age = Date.now() - received;
return Number.isFinite(age) && age >= 0 && age <= ONLINE_MAX_MS
&& (!record?.submission || received > Date.parse(record.submission)) ? report : null;
const received = Date.parse(report.receivedAt),
age = Date.now() - received;
return Number.isFinite(age) &&
age >= 0 &&
age <= ONLINE_MAX_MS &&
(!record?.submission || received > Date.parse(record.submission))
? report
: null;
}
observe = (server: ServerRow) => serverObservation(server, {
provisioning: this.provisioning(server.name), settling: this.settling(server.name),
submission: this.get(server.name)?.submission ?? null, telemetry: this.get(server.name)?.telemetry ?? null
});
observe = (server: ServerRow) =>
serverObservation(server, {
provisioning: this.provisioning(server.name),
settling: this.settling(server.name),
submission: this.get(server.name)?.submission ?? null,
telemetry: this.get(server.name)?.telemetry ?? null
});
patch(server: string, fields: Partial<ServerRow>) {
const record = this.get(server);
if (record) record.server = { ...record.server, ...fields };
src/lib/server/fleet/overview.ts
+176 -54
View File
@@ -15,90 +15,212 @@ export function serverOperations(server: string, now = Date.now()): FleetOperati
function readOperations(now: number, server?: string): FleetOperation[] {
const since = new Date(now - 86_400_000).toISOString();
const bindings = { server: server ?? null, since };
const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`;
const scope = (column: string) => (server === undefined ? '' : `AND ${column} = @server`);
const handle = db();
const operations: FleetOperation[] = [];
const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`)
const slots = handle
.prepare(
`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`
)
.all(bindings) as { server: string; requested_at: string; error: string | null }[];
const recovering = new Map(slots.map(slot => [slot.server, slot]));
const recovering = new Map(slots.map((slot) => [slot.server, slot]));
const representedRecovery = new Set<string>();
const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
const campaigns = handle
.prepare(
`SELECT d.id, d.action, d.started_at, ds.server,
ds.restart_state, ds.error, ds.restarted_at, ds.players
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as {
id: string; action: string; started_at: string; server: string; restart_state: string;
error: string | null; restarted_at: string | null; players: number | null;
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`
)
.all(bindings) as {
id: string;
action: string;
started_at: string;
server: string;
restart_state: string;
error: string | null;
restarted_at: string | null;
players: number | null;
}[];
for (const row of campaigns) {
const state = row.restart_state === 'done' ? 'done'
: ['failed', 'manual'].includes(row.restart_state) ? 'attention' : 'active';
const state =
row.restart_state === 'done'
? 'done'
: ['failed', 'manual'].includes(row.restart_state)
? 'attention'
: 'active';
const slot = recovering.get(row.server);
if (row.restarted_at && state !== 'done') representedRecovery.add(row.server);
const label = state === 'done' ? 'Update complete' : state === 'attention' ? 'Update needs attention'
: row.restart_state === 'held' ? 'Held by a beta'
: row.restarted_at ? 'Recovering' : (row.players ?? 0) > 0 ? 'Waiting for an empty lobby' : 'Queued for restart';
operations.push({ key: `campaign:${row.id}:${row.server}`, group: `campaign:${row.id}`,
kind: 'campaign', server: row.server, label, state, startedAt: row.started_at,
detail: state === 'done' ? 'Update applied and restart checks completed' : row.error ?? (row.restarted_at ? slot?.error ?? 'Verifying server health'
: (row.players ?? 0) > 0 ? `${row.players} players at the last check`
: 'Waiting for restart eligibility and available capacity'),
href: `/campaigns/${encodeURIComponent(row.id)}` });
const label =
state === 'done'
? 'Update complete'
: state === 'attention'
? 'Update needs attention'
: row.restart_state === 'held'
? 'Held by a beta'
: row.restarted_at
? 'Recovering'
: (row.players ?? 0) > 0
? 'Waiting for an empty lobby'
: 'Queued for restart';
operations.push({
key: `campaign:${row.id}:${row.server}`,
group: `campaign:${row.id}`,
kind: 'campaign',
server: row.server,
label,
state,
startedAt: row.started_at,
detail:
state === 'done'
? 'Update applied and restart checks completed'
: (row.error ??
(row.restarted_at
? (slot?.error ?? 'Verifying server health')
: (row.players ?? 0) > 0
? `${row.players} players at the last check`
: 'Waiting for restart eligibility and available capacity')),
href: `/campaigns/${encodeURIComponent(row.id)}`
});
}
// Only the newest lifecycle operation can still require action on a server.
// Separate branches use state/rowid indexes for fleet reads and the server
// index for detail history. Completed work is never loaded for the full fleet.
const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
const lifecycle = handle
.prepare(
`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
${server === undefined ? '' : `UNION
${
server === undefined
? ''
: `UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND server = @server
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`}
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`
}
UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`)
.all(bindings) as { id: string; server: string; action: string; state: string; stage: string;
error: string | null; started_at: string }[];
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`
)
.all(bindings) as {
id: string;
server: string;
action: string;
state: string;
stage: string;
error: string | null;
started_at: string;
}[];
for (const row of lifecycle) {
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
const state = ['done', 'warning', 'cancelled'].includes(row.state) ? 'done'
: ['failed', 'interrupted'].includes(row.state) ? 'attention' : 'active';
operations.push({ key: `deployment:${row.id}`, group: `deployment:${row.id}`, kind: 'deployment',
server: row.server, state, startedAt: row.started_at,
activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`, resultState: row.state,
label: state === 'done' ? row.state === 'cancelled' ? 'Removal cancelled' : row.state === 'warning' ? 'Completed with warnings' : 'Complete'
: state === 'attention' ? `${actionLabels[row.action as ProvisioningAction]} needs attention` : row.stage,
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
const state = ['done', 'warning', 'cancelled'].includes(row.state)
? 'done'
: ['failed', 'interrupted'].includes(row.state)
? 'attention'
: 'active';
operations.push({
key: `deployment:${row.id}`,
group: `deployment:${row.id}`,
kind: 'deployment',
server: row.server,
state,
startedAt: row.started_at,
activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`,
resultState: row.state,
label:
state === 'done'
? row.state === 'cancelled'
? 'Removal cancelled'
: row.state === 'warning'
? 'Completed with warnings'
: 'Complete'
: state === 'attention'
? `${actionLabels[row.action as ProvisioningAction]} needs attention`
: row.stage,
detail: row.error ?? row.stage,
href: `/provisioning/${encodeURIComponent(row.id)}`
});
}
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`)
.all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' };
const betas = handle
.prepare(
`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`
)
.all(bindings) as {
id: number;
opened_at: string;
server: string;
stage: string;
error: string | null;
}[];
const stages: Record<string, string> = {
queued: 'Waiting for an earlier test wave',
deploying: 'Deploying beta',
'awaiting-empty': 'Waiting for an empty lobby',
restarting: 'Restarting',
'awaiting-health': 'Recovering',
'awaiting-match': 'Observing a match',
ready: 'Ready for review',
failed: 'Beta needs attention'
};
for (const row of betas) {
if (['restarting', 'awaiting-health'].includes(row.stage)) representedRecovery.add(row.server);
operations.push({ key: `beta:${row.id}:${row.server}`, group: `beta:${row.id}`,
kind: 'beta', server: row.server, startedAt: row.opened_at,
state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
if (['restarting', 'awaiting-health'].includes(row.stage))
representedRecovery.add(row.server);
operations.push({
key: `beta:${row.id}:${row.server}`,
group: `beta:${row.id}`,
kind: 'beta',
server: row.server,
startedAt: row.opened_at,
state: row.stage === 'failed' ? 'attention' : 'active',
label: stages[row.stage] ?? row.stage,
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`,
href: `/betas/${row.id}`
});
}
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
.all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
const maintenance = handle
.prepare(
`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`
)
.all(bindings) as {
id: number;
server: string;
kind: string;
detail: string | null;
requested_at: string;
}[];
for (const job of maintenance) {
operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance',
server: job.server, state: 'attention', startedAt: job.requested_at,
label: job.kind === 'update' ? 'Update verification needs attention' : 'Maintenance needs attention',
detail: job.detail ?? 'Inspect the maintenance result', href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}` });
operations.push({
key: `maintenance:${job.id}`,
group: `maintenance:${job.id}`,
kind: 'maintenance',
server: job.server,
state: 'attention',
startedAt: job.requested_at,
label:
job.kind === 'update'
? 'Update verification needs attention'
: 'Maintenance needs attention',
detail: job.detail ?? 'Inspect the maintenance result',
href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}`
});
}
for (const slot of slots) {
if (representedRecovery.has(slot.server)) continue;
operations.push({ key: `recovery:${slot.server}:${slot.requested_at}`, group: `recovery:${slot.server}`,
kind: 'recovery', server: slot.server, startedAt: slot.requested_at, state: slot.error ? 'attention' : 'active',
label: 'Recovering from restart', detail: slot.error ?? 'Verifying server health',
href: `/servers/${encodeURIComponent(slot.server)}` });
operations.push({
key: `recovery:${slot.server}:${slot.requested_at}`,
group: `recovery:${slot.server}`,
kind: 'recovery',
server: slot.server,
startedAt: slot.requested_at,
state: slot.error ? 'attention' : 'active',
label: 'Recovering from restart',
detail: slot.error ?? 'Verifying server health',
href: `/servers/${encodeURIComponent(slot.server)}`
});
}
return operations.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
}
src/lib/server/fleet/player-capacity.ts
+4 -1
View File
@@ -3,7 +3,10 @@ import { monitoringStatus } from './health-observations.js';
import { MonitoringSnapshot } from './monitoring-snapshot.js';
/** Read only the accepted, current-instance sample; never dispatch a probe. */
export function observedPlayerCapacity(server: string, snapshot: MonitoringSnapshot): PlayerCapacityObservation {
export function observedPlayerCapacity(
server: string,
snapshot: MonitoringSnapshot
): PlayerCapacityObservation {
const game = monitoringStatus(server, snapshot)?.game;
const capacity = game?.result.ok ? game.result.status.capacity : null;
return typeof capacity === 'number' && Number.isSafeInteger(capacity) && capacity > 0
src/lib/server/fleet/player-profile.ts
+70 -23
View File
@@ -2,7 +2,13 @@ import { db } from '../db/index.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
import { getPersonalBestRecords } from './leaderboard-records.js';
import { countsAsGamePlayed } from './leaderboard-participation.js';
import type { LeaderboardOptions, PlayerProfile, PlayerStats, PlayerMapRecord, PlayerMatch } from '../../leaderboard.js';
import type {
LeaderboardOptions,
PlayerProfile,
PlayerStats,
PlayerMapRecord,
PlayerMatch
} from '../../leaderboard.js';
const totals = `MAX(mp.highest_round) AS highestRound, SUM(mp.kills) AS kills,
SUM(mp.rounds_survived) AS roundsSurvived, SUM(mp.headshots) AS headshots,
@@ -12,38 +18,68 @@ const scope = `FROM leaderboard_match_players mp JOIN leaderboard_matches m ON m
WHERE m.game IN ('t5', 't6') AND m.status = 'completed' AND mp.guid = ? AND mp.has_stats = 1
AND m.highest_round >= ${MIN_RECORDED_ROUND} AND mp.highest_round >= ${MIN_RECORDED_ROUND}`;
export function getPlayerProfile(guid: string, requestedOffset = 0,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players'> = {}): PlayerProfile | null {
export function getPlayerProfile(
guid: string,
requestedOffset = 0,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players'> = {}
): PlayerProfile | null {
const handle = db();
// Keep game-specific ingestion identities, but show one identity with the latest reported name.
const player = handle.prepare(`SELECT guid, name, MIN(first_seen_at) OVER () AS firstSeen,
const player = handle
.prepare(
`SELECT guid, name, MIN(first_seen_at) OVER () AS firstSeen,
MAX(last_seen_at) OVER () AS lastSeen FROM leaderboard_players
WHERE game IN ('t5', 't6') AND guid = ? ORDER BY last_seen_at DESC, game LIMIT 1`)
WHERE game IN ('t5', 't6') AND guid = ? ORDER BY last_seen_at DESC, game LIMIT 1`
)
.get(guid) as Pick<PlayerProfile, 'guid' | 'name' | 'firstSeen' | 'lastSeen'> | undefined;
if (!player) return null;
const { matchCount, ...stats } = handle.prepare(`SELECT COUNT(*) AS matchCount, ${totals} ${scope}`)
const { matchCount, ...stats } = handle
.prepare(`SELECT COUNT(*) AS matchCount, ${totals} ${scope}`)
.get(guid) as PlayerStats & { matchCount: number };
if (!matchCount) return null;
const historyParameters: Array<string | number> = [guid];
let historyScope = scope;
if (options.game) { historyScope += ' AND m.game = ?'; historyParameters.push(options.game); }
if (options.map) { historyScope += ' AND m.map_key = ?'; historyParameters.push(options.map); }
if (options.players) { historyScope += ' AND m.start_players = ?'; historyParameters.push(options.players); }
const matchesTotal = options.game || options.map || options.players
? (handle.prepare(`SELECT COUNT(*) AS total ${historyScope}`).get(...historyParameters) as { total: number }).total
: matchCount;
if (options.game) {
historyScope += ' AND m.game = ?';
historyParameters.push(options.game);
}
if (options.map) {
historyScope += ' AND m.map_key = ?';
historyParameters.push(options.map);
}
if (options.players) {
historyScope += ' AND m.start_players = ?';
historyParameters.push(options.players);
}
const matchesTotal =
options.game || options.map || options.players
? (
handle
.prepare(`SELECT COUNT(*) AS total ${historyScope}`)
.get(...historyParameters) as { total: number }
).total
: matchCount;
const limit = 25;
const lastOffset = Math.max(0, Math.floor((matchesTotal - 1) / limit) * limit);
const offset = Number.isSafeInteger(requestedOffset) ? Math.min(lastOffset, Math.max(0, requestedOffset)) : 0;
const mapStats = handle.prepare(`SELECT m.game, m.map_key AS key, m.map_name AS name,
const offset = Number.isSafeInteger(requestedOffset)
? Math.min(lastOffset, Math.max(0, requestedOffset))
: 0;
const mapStats = handle
.prepare(
`SELECT m.game, m.map_key AS key, m.map_name AS name,
MAX(mp.last_seen_at) AS lastSeen, ${totals} ${scope}
GROUP BY m.game, m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, m.game, key`)
GROUP BY m.game, m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, m.game, key`
)
.all(guid) as Array<Omit<PlayerMapRecord, 'records' | 'ranks'>>;
const [records, ...mapRecords] = getPersonalBestRecords(handle, [{ guid, ...stats },
...mapStats.map(map => ({ guid, ...map, map: map.key }))]);
const [records, ...mapRecords] = getPersonalBestRecords(handle, [
{ guid, ...stats },
...mapStats.map((map) => ({ guid, ...map, map: map.key }))
]);
// Rank all three bests in one pass over this player's maps. Filter the GUID
// only after ranking competitors so ties and skipped ranks match the leaderboard.
const mapRanks = handle.prepare(`WITH totals AS (
const mapRanks = handle
.prepare(
`WITH totals AS (
SELECT requested.key AS requestId, mp.guid,
MAX(mp.highest_round) AS highestRound, MAX(mp.kills) AS bestKills, MAX(mp.best_score) AS bestScore
FROM json_each(?) requested
@@ -60,14 +96,25 @@ export function getPlayerProfile(guid: string, requestedOffset = 0,
RANK() OVER (PARTITION BY requestId ORDER BY bestKills DESC) AS bestKills,
RANK() OVER (PARTITION BY requestId ORDER BY bestScore DESC) AS bestScore
FROM totals
) SELECT highestRound, bestKills, bestScore FROM ranked WHERE guid = ? ORDER BY requestId`)
.all(JSON.stringify(mapStats.map(({ game, key }) => ({ game, key }))), guid) as PlayerMapRecord['ranks'][];
const maps = mapStats.map((map, index) => ({ ...map, records: mapRecords[index], ranks: mapRanks[index] }));
const matches = handle.prepare(`SELECT m.id, m.game, m.server, m.server_deleted_at, m.map_name AS mapName,
) SELECT highestRound, bestKills, bestScore FROM ranked WHERE guid = ? ORDER BY requestId`
)
.all(
JSON.stringify(mapStats.map(({ game, key }) => ({ game, key }))),
guid
) as PlayerMapRecord['ranks'][];
const maps = mapStats.map((map, index) => ({
...map,
records: mapRecords[index],
ranks: mapRanks[index]
}));
const matches = handle
.prepare(
`SELECT m.id, m.game, m.server, m.server_deleted_at, m.map_name AS mapName,
m.started_at AS startedAt, mp.last_seen_at AS lastSeen, m.status, mp.completed,
mp.highest_round AS highestRound, mp.rounds_survived AS roundsSurvived,
mp.kills, mp.headshots, mp.revives, mp.downs, mp.best_score AS bestScore
${historyScope} ORDER BY mp.last_seen_at DESC, m.id DESC LIMIT ? OFFSET ?`)
${historyScope} ORDER BY mp.last_seen_at DESC, m.id DESC LIMIT ? OFFSET ?`
)
.all(...historyParameters, limit, offset) as PlayerMatch[];
return { ...player, stats, records, maps, matches, matchesTotal, offset, limit };
}
src/lib/server/fleet/plugin-releases.ts
+110 -25
View File
@@ -1,13 +1,29 @@
import { createHash } from 'node:crypto';
import { db } from '../db/index.js';
import { createScript, getScript, getVersion, uploadNewVersion, type ScopeType } from './scripts.js';
import {
createScript,
getScript,
getVersion,
uploadNewVersion,
type ScopeType
} from './scripts.js';
import { findRunningBetaForScript } from './beta.js';
import { downloadGithubAsset, getGithubRelease, GithubReleaseError } from '../integrations/github-releases.js';
import {
downloadGithubAsset,
getGithubRelease,
GithubReleaseError
} from '../integrations/github-releases.js';
import type { PluginSource, PluginUpdate } from '$lib/plugin-github.js';
export function pluginSource(pluginId: number): PluginSource | null {
return db().prepare(`SELECT p.* FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
WHERE v.script_id = ? ORDER BY v.version DESC LIMIT 1`).get(pluginId) as PluginSource | undefined ?? null;
return (
(db()
.prepare(
`SELECT p.* FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
WHERE v.script_id = ? ORDER BY v.version DESC LIMIT 1`
)
.get(pluginId) as PluginSource | undefined) ?? null
);
}
export async function checkPluginUpdate(pluginId: number): Promise<PluginUpdate> {
@@ -19,13 +35,27 @@ export async function checkPluginUpdate(pluginId: number): Promise<PluginUpdate>
try {
const release = await getGithubRelease(`https://github.com/${source.repository}`);
// A latest-stable release may already be in history when a newer prerelease is imported.
if (release.id !== source.release_id && Date.parse(release.publishedAt) <= Date.parse(source.published_at)) {
if (
release.id !== source.release_id &&
Date.parse(release.publishedAt) <= Date.parse(source.published_at)
) {
return { ...base, release, status: 'ahead' };
}
const imported = db().prepare(`SELECT p.version_id FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
const imported = db()
.prepare(
`SELECT p.version_id FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
WHERE v.script_id = ? AND lower(p.repository) = lower(?) AND p.release_id = ? AND p.asset_name = ?
ORDER BY v.version DESC LIMIT 1`).get(pluginId, source.repository, release.id, source.asset_name) as { version_id: number } | undefined;
if (imported) return { ...base, release, status: imported.version_id === plugin.current_version_id ? 'current' : 'imported', versionId: imported.version_id };
ORDER BY v.version DESC LIMIT 1`
)
.get(pluginId, source.repository, release.id, source.asset_name) as
{ version_id: number } | undefined;
if (imported)
return {
...base,
release,
status: imported.version_id === plugin.current_version_id ? 'current' : 'imported',
versionId: imported.version_id
};
return { ...base, release, status: 'available' };
} catch (error) {
return { ...base, release: null, status: 'error', error: (error as Error).message };
@@ -42,44 +72,99 @@ export interface ImportPluginReleaseInput {
}
export async function importPluginRelease(input: ImportPluginReleaseInput) {
if (typeof input.url !== 'string' || !Number.isSafeInteger(input.assetId) || input.assetId < 1) {
if (
typeof input.url !== 'string' ||
!Number.isSafeInteger(input.assetId) ||
input.assetId < 1
) {
throw new GithubReleaseError('A release URL and DLL asset selection are required.');
}
if (input.pluginId !== undefined && (!Number.isSafeInteger(input.pluginId) || input.pluginId < 1)) {
if (
input.pluginId !== undefined &&
(!Number.isSafeInteger(input.pluginId) || input.pluginId < 1)
) {
throw new GithubReleaseError('Invalid plugin id.');
}
const requirePlugin = () => {
const plugin = input.pluginId === undefined ? undefined : getScript(input.pluginId);
if (input.pluginId !== undefined && (!plugin || plugin.kind !== 'plugin')) throw new GithubReleaseError('Plugin not found.', 404);
if (plugin && findRunningBetaForScript(plugin.id)) throw new GithubReleaseError('Finish the running beta before importing another release.', 409);
if (input.pluginId !== undefined && (!plugin || plugin.kind !== 'plugin'))
throw new GithubReleaseError('Plugin not found.', 404);
if (plugin && findRunningBetaForScript(plugin.id))
throw new GithubReleaseError(
'Finish the running beta before importing another release.',
409
);
return plugin;
};
requirePlugin();
if (input.pluginId === undefined && (typeof input.name !== 'string' || !input.name.trim())) throw new GithubReleaseError('A plugin name is required.');
if (input.pluginId === undefined && (typeof input.name !== 'string' || !input.name.trim()))
throw new GithubReleaseError('A plugin name is required.');
// Refetch the exact reviewed release: asset names, notes, tags and digests are never trusted from the client.
const release = await getGithubRelease(input.url, true);
const asset = release.assets.find(a => a.id === input.assetId);
if (!asset) throw new GithubReleaseError('That DLL is no longer available in this release. Preview it again.');
const asset = release.assets.find((a) => a.id === input.assetId);
if (!asset)
throw new GithubReleaseError(
'That DLL is no longer available in this release. Preview it again.'
);
const content = await downloadGithubAsset(release, asset);
const sha = createHash('sha256').update(content).digest('hex');
return db().transaction(() => {
const plugin = requirePlugin();
if (plugin) {
const prior = db().prepare(`SELECT v.id FROM script_versions v JOIN plugin_imports p ON p.version_id = v.id
const prior = db()
.prepare(
`SELECT v.id FROM script_versions v JOIN plugin_imports p ON p.version_id = v.id
WHERE v.script_id = ? AND lower(p.repository) = lower(?) AND p.release_id = ? AND p.asset_id = ? AND v.sha256 = ?
ORDER BY v.version DESC LIMIT 1`).get(plugin.id, release.repository, release.id, asset.id, sha) as { id: number } | undefined;
ORDER BY v.version DESC LIMIT 1`
)
.get(plugin.id, release.repository, release.id, asset.id, sha) as
{ id: number } | undefined;
if (prior) return result(plugin.id, getVersion(prior.id)!, true);
}
const created = plugin ? { script: plugin, version: uploadNewVersion(plugin.id, content, release.notes, release.tag) }
: createScript({ name: input.name!.trim(), kind: 'plugin', targetRelPath: asset.name, content,
releaseVersion: release.tag, notes: release.notes, scopeType: input.scopeType, scopeValue: input.scopeValue });
db().prepare(`INSERT INTO plugin_imports (version_id, repository, release_id, release_url, asset_id, asset_name, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(created.version.id, release.repository, release.id, release.url, asset.id, asset.name, release.publishedAt);
const created = plugin
? {
script: plugin,
version: uploadNewVersion(plugin.id, content, release.notes, release.tag)
}
: createScript({
name: input.name!.trim(),
kind: 'plugin',
targetRelPath: asset.name,
content,
releaseVersion: release.tag,
notes: release.notes,
scopeType: input.scopeType,
scopeValue: input.scopeValue
});
db()
.prepare(
`INSERT INTO plugin_imports (version_id, repository, release_id, release_url, asset_id, asset_name, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`
)
.run(
created.version.id,
release.repository,
release.id,
release.url,
asset.id,
asset.name,
release.publishedAt
);
return result(created.script.id, created.version, false);
})();
}
function result(pluginId: number, version: NonNullable<ReturnType<typeof getVersion>>, alreadyImported: boolean) {
return { pluginId, versionId: version.id, releaseVersion: version.release_version, sha256: version.sha256,
alreadyImported, betaUrl: `/betas?pluginId=${pluginId}&versionId=${version.id}` };
function result(
pluginId: number,
version: NonNullable<ReturnType<typeof getVersion>>,
alreadyImported: boolean
) {
return {
pluginId,
versionId: version.id,
releaseVersion: version.release_version,
sha256: version.sha256,
alreadyImported,
betaUrl: `/betas?pluginId=${pluginId}&versionId=${version.id}`
};
}
src/lib/server/fleet/plutonium.ts
+1 -1
View File
@@ -1,2 +1,2 @@
// Compatibility for existing jobs and API clients.
export * from "../maintenance/runtime.js";
export * from '../maintenance/runtime.js';
src/lib/server/fleet/promoted-artifacts.ts
+12 -4
View File
@@ -1,10 +1,18 @@
import { getVersion, getVersionContent, getVersionMetadata, listScripts } from './scripts.js';
function selectedArtifacts<Version>(game: string | null, map: string | null, readVersion: (id: number) => Version | undefined) {
function selectedArtifacts<Version>(
game: string | null,
map: string | null,
readVersion: (id: number) => Version | undefined
) {
return (['script', 'plugin'] as const).flatMap((kind) =>
listScripts(kind).flatMap((script) => {
const scopeMatches = script.scope_type === 'game' ? script.scope_value === game
: script.scope_type === 'map' ? script.scope_value === map : false;
const scopeMatches =
script.scope_type === 'game'
? script.scope_value === game
: script.scope_type === 'map'
? script.scope_value === map
: false;
if (!scopeMatches || !script.current_version_id) return [];
const version = readVersion(script.current_version_id);
return version ? [{ kind, script, version }] : [];
@@ -14,7 +22,7 @@ function selectedArtifacts<Version>(game: string | null, map: string | null, rea
/** Published file artifacts shared by provisioning and drift reconciliation. */
export function promotedArtifacts(game: string | null, map: string | null) {
return selectedArtifacts(game, map, id => {
return selectedArtifacts(game, map, (id) => {
const version = getVersion(id);
return version ? { ...version, content: getVersionContent(id) } : undefined;
});
src/lib/server/fleet/provisioning.ts
+25 -8
View File
@@ -2,17 +2,34 @@ import { db } from '../db/index.js';
/** Maintenance must leave pending lifecycle work and retained recovery to Provisioning. */
export function hasActiveProvisioning(server: string, exceptOperationId = ''): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments
return (
!!db()
.prepare(
`SELECT 1 FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL AND id != ?
AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1`).get(server, exceptOperationId)
|| !!db().prepare(`SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL
AND (operation_id IS NULL OR operation_id != ?)`).get(server, exceptOperationId);
AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1`
)
.get(server, exceptOperationId) ||
!!db()
.prepare(
`SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL
AND (operation_id IS NULL OR operation_id != ?)`
)
.get(server, exceptOperationId)
);
}
/** Only the latest lifecycle operation can describe the current server. */
export function isProvisioning(server: string): boolean {
const operation = db().prepare(`SELECT action, state FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as
{ action: string; state: string } | undefined;
return !!operation && ['create', 'rebuild', 'start'].includes(operation.action) && ['queued', 'running', 'awaiting-health'].includes(operation.state);
const operation = db()
.prepare(
`SELECT action, state FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`
)
.get(server) as { action: string; state: string } | undefined;
return (
!!operation &&
['create', 'rebuild', 'start'].includes(operation.action) &&
['queued', 'running', 'awaiting-health'].includes(operation.state)
);
}
src/lib/server/fleet/rollback.ts
+148 -50
View File
@@ -3,7 +3,12 @@ import type { ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
import { getVersion } from './scripts.js';
import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js';
import {
cancelStagedFiles,
isFileApplied,
queueFileChange,
stagedFileChanges
} from './staged-files.js';
// Roll a promoted deployment back to whatever versions were current before
// the promote landed. Reads deployment_scripts as the source of truth for
@@ -12,7 +17,15 @@ import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges }
// pre-multi-script legacy rows.
export class RollbackError extends Error {
constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back' | 'snapshot-pending') {
constructor(
message: string,
public readonly kind:
| 'history-expired'
| 'no-such-deployment'
| 'not-promote'
| 'already-rolled-back'
| 'snapshot-pending'
) {
super(message);
}
}
@@ -58,31 +71,53 @@ function nowIso(): string {
}
function rollbackDeploymentId(promoteId: string): string {
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
const stamp = new Date()
.toISOString()
.replace(/[-:]/g, '')
.replace(/\.\d+Z$/, 'Z');
return `dep-${stamp}-rollback-${promoteId}`;
}
export interface RollbackDeploymentResult {
deploymentId: string; // the new rollback deployment
reverses: string; // the promote deployment id
scriptIds: number[]; // scripts whose current_version_id got reverted
deploymentId: string; // the new rollback deployment
reverses: string; // the promote deployment id
scriptIds: number[]; // scripts whose current_version_id got reverted
affectedServers: string[];
}
export async function rollbackPromoteDeployment(promoteId: string): Promise<RollbackDeploymentResult> {
export async function rollbackPromoteDeployment(
promoteId: string
): Promise<RollbackDeploymentResult> {
const dep = db()
.prepare(`SELECT id, action, script_id, script_version_id, prior_version_id, rolled_back_by, beta_id
FROM deployments WHERE id = ?`)
.prepare(
`SELECT id, action, script_id, script_version_id, prior_version_id, rolled_back_by, beta_id
FROM deployments WHERE id = ?`
)
.get(promoteId) as DeploymentRow | undefined;
if (!dep) throw new RollbackError(`no such deployment ${promoteId}`, 'no-such-deployment');
if (db().prepare("SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?").get(promoteId)) {
throw new RollbackError('Rollback file contents have expired under the retention policy.', 'history-expired');
if (
db()
.prepare(
"SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?"
)
.get(promoteId)
) {
throw new RollbackError(
'Rollback file contents have expired under the retention policy.',
'history-expired'
);
}
if (dep.action !== 'promote') {
throw new RollbackError(`deployment ${promoteId} is action='${dep.action}', not 'promote'`, 'not-promote');
throw new RollbackError(
`deployment ${promoteId} is action='${dep.action}', not 'promote'`,
'not-promote'
);
}
if (dep.rolled_back_by) {
throw new RollbackError(`deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`, 'already-rolled-back');
throw new RollbackError(
`deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`,
'already-rolled-back'
);
}
// Source of truth for which scripts (and prior versions) this promote
@@ -92,13 +127,15 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
.prepare(`SELECT * FROM deployment_scripts WHERE deployment_id = ? ORDER BY script_id`)
.all(promoteId) as DepScriptRow[];
if (scripts.length === 0 && dep.script_id !== null && dep.script_version_id !== null) {
scripts = [{
deployment_id: promoteId,
script_id: dep.script_id,
script_version_id: dep.script_version_id,
prior_version_id: dep.prior_version_id,
kind: 'script'
}];
scripts = [
{
deployment_id: promoteId,
script_id: dep.script_id,
script_version_id: dep.script_version_id,
prior_version_id: dep.prior_version_id,
kind: 'script'
}
];
}
const fileRows = db()
@@ -132,17 +169,29 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
const staged = stagedFileChanges('campaign', promoteId);
const scriptForFile = (file: FileRow): DepScriptRow | undefined => {
const candidates = scriptsByTarget.get(file.rel_path) ?? [];
const versionId = staged.find(row => row.deployment_file_id === file.id)?.script_version_id;
if (versionId) return candidates.find(script => script.script_version_id === versionId);
const versionId = staged.find(
(row) => row.deployment_file_id === file.id
)?.script_version_id;
if (versionId) return candidates.find((script) => script.script_version_id === versionId);
if (dep.beta_id) {
const participant = candidates.find(script => db().prepare(`SELECT 1 FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`).get(dep.beta_id, file.server, script.script_id));
const participant = candidates.find((script) =>
db()
.prepare(
`SELECT 1 FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`
)
.get(dep.beta_id, file.server, script.script_id)
);
if (participant) return participant;
}
return candidates.find(script => getVersion(script.script_version_id)?.sha256 === file.sha256_after) ?? candidates[0];
return (
candidates.find(
(script) => getVersion(script.script_version_id)?.sha256 === file.sha256_after
) ?? candidates[0]
);
};
const kindForFile = (file: FileRow): ArtifactKind => {
const stagedKind = staged.find(row => row.deployment_file_id === file.id)?.kind;
const stagedKind = staged.find((row) => row.deployment_file_id === file.id)?.kind;
if (stagedKind) return stagedKind;
const owner = scriptForFile(file);
if (owner) return owner.kind ?? kindByScriptId.get(owner.script_id) ?? 'script';
@@ -164,11 +213,23 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
// must not restore or remove files that it never installed. Rows from
// older releases have no staging record and retain their audit meaning.
const kind = kindForFile(f);
const preservesBetaSnapshot = staged.some(row => row.server === f.server && row.kind === kind
&& row.rel_path === f.rel_path && row.preserve_before_image);
if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue;
const preservesBetaSnapshot = staged.some(
(row) =>
row.server === f.server &&
row.kind === kind &&
row.rel_path === f.rel_path &&
row.preserve_before_image
);
if (
!preservesBetaSnapshot &&
isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false
)
continue;
if (f.snapshot_pending) {
throw new RollbackError(`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`, 'snapshot-pending');
throw new RollbackError(
`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`,
'snapshot-pending'
);
}
const arr = perServer.get(f.server);
if (arr) arr.push(f);
@@ -176,8 +237,13 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
}
const affectedServers: string[] = [];
const active = (server: string) => !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server);
const active = (server: string) =>
!!db()
.prepare(
`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`
)
.get(promoteId, server);
// Write the new rollback deployment + mark the original.
const insertDep = db().prepare(
@@ -204,8 +270,12 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
cancelStagedFiles('campaign', promoteId);
// Withdrawing this source leaves a shared restart available to other
// campaigns. Provisioning cancels an operation with no eligible sources.
db().prepare(`UPDATE deployment_servers SET restart_state = 'skipped', error = NULL
WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).run(promoteId);
db()
.prepare(
`UPDATE deployment_servers SET restart_state = 'skipped', error = NULL
WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`
)
.run(promoteId);
insertDep.run(newDepId, startedAt);
// Mirror the promote's script list — swapped: the "current" version
// after rollback is the promote's prior; the "prior" of the rollback
@@ -221,16 +291,20 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
}
for (const [server, files] of perServer) {
if (!active(server)) continue;
if (files.some(file => file.verb !== 'added' && file.content_before === null)) {
insertDepServer.run(newDepId, server, 0, 'failed', 'Original file snapshot is missing; cannot restore safely');
if (files.some((file) => file.verb !== 'added' && file.content_before === null)) {
insertDepServer.run(
newDepId,
server,
0,
'failed',
'Original file snapshot is missing; cannot restore safely'
);
continue;
}
insertDepServer.run(newDepId, server, files.length, 'pending', null);
for (const f of files) {
const invertedVerb =
f.verb === 'added' ? 'removed'
: f.verb === 'removed' ? 'added'
: 'updated';
f.verb === 'added' ? 'removed' : f.verb === 'removed' ? 'added' : 'updated';
const inserted = insertDepFile.run(
newDepId,
server,
@@ -242,27 +316,51 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
putFileBlob(f.content_before)
);
const owningScript = scriptForFile(f);
const betaPrior = dep.beta_id && owningScript ? db().prepare(`SELECT prior_version_id FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`).get(dep.beta_id, server, owningScript.script_id) as
{ prior_version_id: number | null } | undefined : undefined;
const priorVersionId = betaPrior ? betaPrior.prior_version_id : owningScript?.prior_version_id;
const betaPrior =
dep.beta_id && owningScript
? (db()
.prepare(
`SELECT prior_version_id FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`
)
.get(dep.beta_id, server, owningScript.script_id) as
{ prior_version_id: number | null } | undefined)
: undefined;
const priorVersionId = betaPrior
? betaPrior.prior_version_id
: owningScript?.prior_version_id;
const priorVersion = priorVersionId ? getVersion(priorVersionId) : null;
queueFileChange({ sourceKind: 'campaign', sourceId: newDepId, server,
kind: kindForFile(f), relPath: f.rel_path,
queueFileChange({
sourceKind: 'campaign',
sourceId: newDepId,
server,
kind: kindForFile(f),
relPath: f.rel_path,
content: f.verb === 'added' ? null : f.content_before,
scriptVersionId: f.verb === 'updated' ? priorVersion?.id ?? null : null,
inventorySource: `rollback:${newDepId}`, deploymentFileId: Number(inserted.lastInsertRowid) });
scriptVersionId: f.verb === 'updated' ? (priorVersion?.id ?? null) : null,
inventorySource: `rollback:${newDepId}`,
deploymentFileId: Number(inserted.lastInsertRowid)
});
}
affectedServers.push(server);
}
db().prepare(`UPDATE deployments SET rolled_back_by = ? WHERE id = ?`).run(newDepId, promoteId);
db()
.prepare(`UPDATE deployments SET rolled_back_by = ? WHERE id = ?`)
.run(newDepId, promoteId);
// Restore each affected script's current_version_id.
for (const s of scripts) {
db().prepare(`UPDATE scripts SET current_version_id = ? WHERE id = ?`).run(s.prior_version_id, s.script_id);
db()
.prepare(`UPDATE scripts SET current_version_id = ? WHERE id = ?`)
.run(s.prior_version_id, s.script_id);
}
})();
emit({ type: 'deployment-opened', deploymentId: newDepId, action: 'rollback', servers: affectedServers });
emit({
type: 'deployment-opened',
deploymentId: newDepId,
action: 'rollback',
servers: affectedServers
});
return {
deploymentId: newDepId,
src/lib/server/fleet/runtime-config.ts
+6 -1
View File
@@ -19,7 +19,12 @@ export function applyRuntimePassword(text: string, password?: string | null): st
}
if (!matched) {
if (lines.at(-1) === '') lines.pop();
lines.push('', '// Added by Plutainer from PLUTAINER_RCON_PASSWORD.', `set rcon_password ${replacement}`, '');
lines.push(
'',
'// Added by Plutainer from PLUTAINER_RCON_PASSWORD.',
`set rcon_password ${replacement}`,
''
);
changed = true;
}
return changed ? lines.join('\n') : text;

Some files were not shown because too many files have changed in this diff Show More