Add Prettier and ESLint tooling (Refs #77)

This commit is contained in:
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
+88 -24
View File
@@ -1,6 +1,12 @@
import { hostOperationTitles } from '../host-operation-labels.js';
import type Database from 'better-sqlite3';
import { normalizeActivityQuery, type ActivityPage, type ActivityQuery, type ActivityRow, type ActivitySource } from '../activity.js';
import {
normalizeActivityQuery,
type ActivityPage,
type ActivityQuery,
type ActivityRow,
type ActivitySource
} from '../activity.js';
import { serverTitle } from '../ui/server-labels.js';
import { db } from './db/index.js';
import { ensureHostSchema } from './hosts/schema.js';
@@ -9,16 +15,22 @@ import { importantOperationSql } from './hosts/observations.js';
const initialized = new WeakSet<Database.Database>();
const quote = (value: string) => `'${value.replaceAll("'", "''")}'`;
const labelCase = (column: string, labels: Record<string, string>, fallback: string) =>
`CASE ${column} ${Object.entries(labels).map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`).join(' ')} ELSE ${quote(fallback)} END`;
`CASE ${column} ${Object.entries(labels)
.map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`)
.join(' ')} ELSE ${quote(fallback)} END`;
// Old attempts may no longer be the native_provisioning pointer. Their stable
// idempotency keys still identify the same retained controller operation.
const lifecycleKey = `CASE
WHEN substr(o.idempotency_key,-6)=':stage' THEN substr(o.idempotency_key,1,length(o.idempotency_key)-6)
${[':migration-create:', ':migrate-remove:'].map(marker => `WHEN instr(o.idempotency_key,${quote(marker)})>0
${[':migration-create:', ':migrate-remove:']
.map(
(marker) => `WHEN instr(o.idempotency_key,${quote(marker)})>0
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length})<>''
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length}) NOT GLOB '*[^0-9]*'
THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`).join('\n')}
THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`
)
.join('\n')}
ELSE o.idempotency_key END`;
/** One row per user-level operation or finding; no command payloads or observations are read. */
@@ -128,19 +140,50 @@ normalized AS (
)
`;
type SqlRow = Omit<ActivityRow, 'source' | 'sources' | 'href'> & { sources: string; route: string; recordId: string; secondaryHostId: string | null };
const sourceOrder: ActivitySource[] = ['maintenance', 'recovery', 'campaign', 'beta', 'autoscaling', 'provisioning', 'host'];
type SqlRow = Omit<ActivityRow, 'source' | 'sources' | 'href'> & {
sources: string;
route: string;
recordId: string;
secondaryHostId: string | null;
};
const sourceOrder: ActivitySource[] = [
'maintenance',
'recovery',
'campaign',
'beta',
'autoscaling',
'provisioning',
'host'
];
// Use the same membership rule for Maintenance history and its exclusion from the main log.
const maintenanceMembershipSql = "(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)";
const maintenanceMembershipSql =
"(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)";
function present(row: SqlRow): ActivityRow {
const sources = sourceOrder.filter(source => row.sources.includes(`|${source}|`));
const href = row.route === 'maintenance-job' ? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}`
: row.route === 'recovery' ? `/maintenance/${encodeURIComponent(row.recordId)}`
: row.route === 'host' ? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all`
: `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`;
return { id: row.id, at: row.at, type: row.type, source: sources[0] ?? 'provisioning', sources,
state: row.state, rawState: row.rawState, title: row.title, detail: row.detail,
server: row.server, serverDeletedAt: row.serverDeletedAt, hostId: row.hostId, hostName: row.hostName, href };
const sources = sourceOrder.filter((source) => row.sources.includes(`|${source}|`));
const href =
row.route === 'maintenance-job'
? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}`
: row.route === 'recovery'
? `/maintenance/${encodeURIComponent(row.recordId)}`
: row.route === 'host'
? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all`
: `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`;
return {
id: row.id,
at: row.at,
type: row.type,
source: sources[0] ?? 'provisioning',
sources,
state: row.state,
rawState: row.rawState,
title: row.title,
detail: row.detail,
server: row.server,
serverDeletedAt: row.serverDeletedAt,
hostId: row.hostId,
hostName: row.hostName,
href
};
}
/** All predicates, totals and pagination operate on the same retained SQL union. */
@@ -149,7 +192,9 @@ export function activityHistory(input: ActivityQuery = {}, now = Date.now()): Ac
const handle = db();
ensureHostSchema(handle);
if (!initialized.has(handle)) {
handle.function('activity_server_title', { deterministic: true }, (name: unknown) => typeof name === 'string' ? serverTitle(name) : '');
handle.function('activity_server_title', { deterministic: true }, (name: unknown) =>
typeof name === 'string' ? serverTitle(name) : ''
);
initialized.add(handle);
}
const conditions: string[] = [];
@@ -158,25 +203,44 @@ export function activityHistory(input: ActivityQuery = {}, now = Date.now()): Ac
if (query.excludeProvisioning) conditions.push("route <> 'provisioning' AND type <> 'restart'");
if (query.excludeMaintenance) conditions.push(`NOT ${maintenanceMembershipSql}`);
const bindings: Record<string, string | number> = {};
for (const key of ['server', 'type', 'state'] as const) if (query[key]) { conditions.push(`${key}=@${key}`); bindings[key] = query[key]; }
if (query.host) { conditions.push('(hostId=@host OR secondaryHostId=@host)'); bindings.host = query.host; }
for (const key of ['server', 'type', 'state'] as const)
if (query[key]) {
conditions.push(`${key}=@${key}`);
bindings[key] = query[key];
}
if (query.host) {
conditions.push('(hostId=@host OR secondaryHostId=@host)');
bindings.host = query.host;
}
if (query.source) {
// Maintenance is the user-facing umbrella; Recovery remains available as a narrower filter.
conditions.push(query.source === 'maintenance' ? maintenanceMembershipSql : 'instr(sources,@source)>0');
conditions.push(
query.source === 'maintenance' ? maintenanceMembershipSql : 'instr(sources,@source)>0'
);
if (query.source !== 'maintenance') bindings.source = `|${query.source}|`;
}
if (query.days) { conditions.push('julianday(at)>=julianday(@since)'); bindings.since = new Date(now - query.days * 86_400_000).toISOString(); }
if (query.days) {
conditions.push('julianday(at)>=julianday(@since)');
bindings.since = new Date(now - query.days * 86_400_000).toISOString();
}
if (query.q) {
conditions.push("instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0");
conditions.push(
"instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0"
);
bindings.q = query.q.toLocaleLowerCase();
}
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
const limit = query.limit!;
return handle.transaction(() => {
const { total } = handle.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`).get(bindings) as { total: number };
const { total } = handle
.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`)
.get(bindings) as { total: number };
const offset = Math.min(query.offset!, Math.max(0, Math.ceil(total / limit) - 1) * limit);
const rows = handle.prepare(`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId
FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`)
const rows = handle
.prepare(
`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId
FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`
)
.all({ ...bindings, limit, offset }) as SqlRow[];
return { rows: rows.map(present), total, limit, offset, query: { ...query, offset } };
})();
+50 -16
View File
@@ -1,8 +1,10 @@
import type { UserRole } from '../../auth.js';
export type RoutePermission = 'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin';
export type RoutePermission =
'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin';
const within = (route: string, base: string): boolean => route === base || route.startsWith(base + '/');
const within = (route: string, base: string): boolean =>
route === base || route.startsWith(base + '/');
const readMethod = (method: string): boolean => method === 'GET' || method === 'HEAD';
/** Classify SvelteKit's resolved route ID, never a user-supplied URL pathname.
@@ -15,32 +17,63 @@ export function routePermission(routeId: string | null, method: string): RoutePe
if (routeId === null) return reading ? 'viewer' : 'operator';
// These cannot occur in this application's resolved route IDs. Fail closed
// if a future caller accidentally supplies a raw or noncanonical pathname.
if (!routeId.startsWith('/') || /[%\\?#\x00-\x1f\x7f]/.test(routeId)
|| routeId !== '/' && routeId.split('/').slice(1).some(part => !part || part === '.' || part === '..')) return 'admin';
if (
!routeId.startsWith('/') ||
// eslint-disable-next-line no-control-regex -- Reject control characters in resolved route IDs.
/[%\\?#\x00-\x1f\x7f]/.test(routeId) ||
(routeId !== '/' &&
routeId
.split('/')
.slice(1)
.some((part) => !part || part === '.' || part === '..'))
)
return 'admin';
if (within(routeId, '/api/agent')) return 'machine';
if (routeId === '/api/telemetry' && method === 'POST') return 'machine';
if (routeId === '/api/leaderboard' && reading) return 'public';
if ((routeId === '/login' || routeId === '/setup') && reading) return 'public';
if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST') return 'public';
if (routeId === '/account' || ['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId)) return 'authenticated';
if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST')
return 'public';
if (
routeId === '/account' ||
['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId)
)
return 'authenticated';
if (within(routeId, '/api/users') || within(routeId, '/settings') || within(routeId, '/api/settings') || within(routeId, '/api/auth')
|| within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin';
if (
within(routeId, '/api/users') ||
within(routeId, '/settings') ||
within(routeId, '/api/settings') ||
within(routeId, '/api/auth') ||
within(routeId, '/api/server-keys') ||
within(routeId, '/server-keys') ||
within(routeId, '/keys')
)
return 'admin';
if (!reading && ['/api/hosts', '/api/game-files'].some(base => within(routeId, base))) return 'admin';
if (!reading && ['/api/hosts', '/api/game-files'].some((base) => within(routeId, base)))
return 'admin';
if (routeId === '/api/provisioning/[id]/override') return 'admin';
// These endpoints mix operational actions with global policy and schedule
// changes. Per-incident/job actions remain available to operators.
if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) return 'admin';
if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId))
return 'admin';
// Summaries remain visible to viewers; raw configuration, historical file
// bytes, and unfiltered diagnostics may contain runtime credentials.
if (routeId === '/configs/[id]' || routeId === '/api/configs/[id]' || routeId === '/api/configs/inspect'
|| routeId === '/api/campaigns/[id]/file/[fileId]' || routeId === '/api/deployments/[id]/file/[fileId]'
|| routeId === '/api/betas/[id]/revisions/[revision]/files/[scriptId]'
|| routeId === '/api/servers/[name]/logs/[file]' || routeId === '/api/servers/[name]/telemetry-requests'
|| routeId === '/api/servers/[name]/rcon-diagnostics') return 'operator';
if (
routeId === '/configs/[id]' ||
routeId === '/api/configs/[id]' ||
routeId === '/api/configs/inspect' ||
routeId === '/api/campaigns/[id]/file/[fileId]' ||
routeId === '/api/deployments/[id]/file/[fileId]' ||
routeId === '/api/betas/[id]/revisions/[revision]/files/[scriptId]' ||
routeId === '/api/servers/[name]/logs/[file]' ||
routeId === '/api/servers/[name]/telemetry-requests' ||
routeId === '/api/servers/[name]/rcon-diagnostics'
)
return 'operator';
return reading ? 'viewer' : 'operator';
}
@@ -49,7 +82,8 @@ export function routePermission(routeId: string | null, method: string): RoutePe
export function roleAllows(role: UserRole, permission: RoutePermission): boolean {
if (!['viewer', 'operator', 'admin'].includes(role)) return false;
if (permission === 'machine') return false;
if (permission === 'public' || permission === 'authenticated' || permission === 'viewer') return true;
if (permission === 'public' || permission === 'authenticated' || permission === 'viewer')
return true;
if (permission === 'operator') return role === 'operator' || role === 'admin';
return permission === 'admin' && role === 'admin';
}
+174 -52
View File
@@ -1,112 +1,234 @@
import { error, json, redirect, type Handle } from '@sveltejs/kit';
import { readSession, readSessionHash, recordAudit, setupRequired } from './index.js';
import { authBody, authError, requireSameOrigin, requireUserTransport, sessionCookie } from './http.js';
import {
authBody,
authError,
requireSameOrigin,
requireUserTransport,
sessionCookie
} from './http.js';
import { roleAllows, routePermission, type RoutePermission } from './access.js';
import { withRequestUsername } from './request-context.js';
function publicAsset(routeId: string | null, path: string, method: string) {
return routeId === null && ['GET','HEAD'].includes(method) &&
(path.startsWith('/_app/') || path === '/favicon.svg' || ['/agent/ec-agent.py','/agent/ec-agent-update.py','/agent/install.sh','/agent/manifest.json'].includes(path));
return (
routeId === null &&
['GET', 'HEAD'].includes(method) &&
(path.startsWith('/_app/') ||
path === '/favicon.svg' ||
[
'/agent/ec-agent.py',
'/agent/ec-agent-update.py',
'/agent/install.sh',
'/agent/manifest.json'
].includes(path))
);
}
/** Streaming responses must lose access when the underlying session does. */
export function protectEventStream(response: Response, tokenHash: string, permission: RoutePermission, signal: AbortSignal): Response {
export function protectEventStream(
response: Response,
tokenHash: string,
permission: RoutePermission,
signal: AbortSignal
): Response {
if (!response.body) return response;
const reader = response.body.getReader();
let timer: ReturnType<typeof setInterval> | undefined;
let closed = false;
let closeStream: () => void = () => {};
const cleanup = () => { if (timer) clearInterval(timer); signal.removeEventListener('abort', closeStream); };
const cleanup = () => {
if (timer) clearInterval(timer);
signal.removeEventListener('abort', closeStream);
};
const stream = new ReadableStream<Uint8Array>({
start(controller) {
closeStream = () => {
if (closed) return; closed = true; cleanup();
void reader.cancel().catch(() => {}); controller.close();
if (closed) return;
closed = true;
cleanup();
void reader.cancel().catch(() => {});
controller.close();
};
timer = setInterval(() => {
try {
const current = readSessionHash(tokenHash);
if (!current || current.user.mustChangePassword || !roleAllows(current.user.role,permission)) closeStream();
} catch { closeStream(); }
},15_000);
signal.addEventListener('abort',closeStream,{once:true});
if (signal.aborted) { closeStream(); return; }
if (
!current ||
current.user.mustChangePassword ||
!roleAllows(current.user.role, permission)
)
closeStream();
} catch {
closeStream();
}
}, 15_000);
signal.addEventListener('abort', closeStream, { once: true });
if (signal.aborted) {
closeStream();
return;
}
},
async pull(controller) {
try {
const next = await reader.read();
if (closed) return;
if (next.done) { closed=true; cleanup(); controller.close(); }
else controller.enqueue(next.value);
} catch (cause) { if (!closed) { closed=true; cleanup(); controller.error(cause); } }
if (next.done) {
closed = true;
cleanup();
controller.close();
} else controller.enqueue(next.value);
} catch (cause) {
if (!closed) {
closed = true;
cleanup();
controller.error(cause);
}
}
},
cancel() { closed=true; cleanup(); return reader.cancel(); }
cancel() {
closed = true;
cleanup();
return reader.cancel();
}
});
return new Response(stream, {
status: response.status,
statusText: response.statusText,
headers: response.headers
});
return new Response(stream,{status:response.status,statusText:response.statusText,headers:response.headers});
}
export const authenticationHandle: Handle = async ({ event, resolve }) => {
event.locals.user = null; event.locals.sessionHash = null;
event.locals.user = null;
event.locals.sessionHash = null;
const routeId = event.route.id;
const method = event.request.method.toUpperCase();
if (publicAsset(routeId,event.url.pathname,method)) return resolve(event);
const permission = routePermission(routeId,method);
if (!routeId?.startsWith('/api/agent/') && /^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization')??'')) {
return json({error:'Host credentials only authorize the host agent API.'},{status:403});
if (publicAsset(routeId, event.url.pathname, method)) return resolve(event);
const permission = routePermission(routeId, method);
if (
!routeId?.startsWith('/api/agent/') &&
/^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization') ?? '')
) {
return json(
{ error: 'Host credentials only authorize the host agent API.' },
{ status: 403 }
);
}
if (permission === 'machine') return resolve(event);
const api = routeId?.startsWith('/api/') || event.url.pathname.startsWith('/api/');
try {
requireUserTransport(event.url);
if (!['GET','HEAD','OPTIONS'].includes(method)) requireSameOrigin(event.request,event.url);
} catch (cause) { if (api) return authError(cause); throw cause; }
if (!['GET', 'HEAD', 'OPTIONS'].includes(method))
requireSameOrigin(event.request, event.url);
} catch (cause) {
if (api) return authError(cause);
throw cause;
}
const session = readSession(event.cookies.get(sessionCookie(event.url)));
event.locals.user = session?.user??null;
event.locals.sessionHash = session?.tokenHash??null;
event.locals.user = session?.user ?? null;
event.locals.sessionHash = session?.tokenHash ?? null;
const setup = setupRequired();
if (permission !== 'public') {
if (!session) {
if (api) return json({error:setup?'Complete administrator setup before using the management API.':'Sign in to continue.',setupRequired:setup},{status:401,headers:{'Cache-Control':'no-store'}});
redirect(303,setup?'/setup':`/login?next=${encodeURIComponent(event.url.pathname+event.url.search)}`);
if (api)
return json(
{
error: setup
? 'Complete administrator setup before using the management API.'
: 'Sign in to continue.',
setupRequired: setup
},
{ status: 401, headers: { 'Cache-Control': 'no-store' } }
);
redirect(
303,
setup
? '/setup'
: `/login?next=${encodeURIComponent(event.url.pathname + event.url.search)}`
);
}
if (session.user.mustChangePassword && permission !== 'authenticated') {
if (api) return json({error:'Change your temporary password from Account before continuing.',mustChangePassword:true},{status:403,headers:{'Cache-Control':'no-store'}});
redirect(303,'/account');
if (api)
return json(
{
error: 'Change your temporary password from Account before continuing.',
mustChangePassword: true
},
{ status: 403, headers: { 'Cache-Control': 'no-store' } }
);
redirect(303, '/account');
}
if (!roleAllows(session.user.role,permission)) {
const message = permission==='admin'?'Administrator access is required.':'Operator access is required for this action.';
if (api) return json({error:message},{status:403,headers:{'Cache-Control':'no-store'}});
error(403,message);
if (!roleAllows(session.user.role, permission)) {
const message =
permission === 'admin'
? 'Administrator access is required.'
: 'Operator access is required for this action.';
if (api)
return json(
{ error: message },
{ status: 403, headers: { 'Cache-Control': 'no-store' } }
);
error(403, message);
}
// Launch APIs can also save credentials. Operators may launch with saved
// keys, but only administrators may introduce or replace server keys.
if (session.user.role === 'operator' && method === 'POST' && ['/api/servers','/api/deployment-collections'].includes(routeId??'')) {
if (
session.user.role === 'operator' &&
method === 'POST' &&
['/api/servers', '/api/deployment-collections'].includes(routeId ?? '')
) {
try {
const body = await authBody(event.request.clone(),1024*1024);
if (body.plutoKey || (body.keys && typeof body.keys==='object' && Object.values(body.keys).some(Boolean))) {
return json({error:'An administrator must save server keys before an operator launches these servers.'},{status:403});
const body = await authBody(event.request.clone(), 1024 * 1024);
if (
body.plutoKey ||
(body.keys &&
typeof body.keys === 'object' &&
Object.values(body.keys).some(Boolean))
) {
return json(
{
error: 'An administrator must save server keys before an operator launches these servers.'
},
{ status: 403 }
);
}
} catch (cause) { return authError(cause); }
} catch (cause) {
return authError(cause);
}
}
} else if (['/login','/setup'].includes(routeId??'')) {
if (session) redirect(303,session.user.mustChangePassword?'/account':'/');
if (routeId==='/setup'&&!setup) redirect(303,'/login');
if (routeId==='/login'&&setup) redirect(303,'/setup');
} else if (['/login', '/setup'].includes(routeId ?? '')) {
if (session) redirect(303, session.user.mustChangePassword ? '/account' : '/');
if (routeId === '/setup' && !setup) redirect(303, '/login');
if (routeId === '/login' && setup) redirect(303, '/setup');
}
let response = await withRequestUsername(session?.user.username ?? null, () => resolve(event));
if (session && !['GET','HEAD','OPTIONS'].includes(method) && response.status < 400) {
recordAudit(session.user.id,'request',routeId,{method,status:response.status,path:event.url.pathname});
if (session && !['GET', 'HEAD', 'OPTIONS'].includes(method) && response.status < 400) {
recordAudit(session.user.id, 'request', routeId, {
method,
status: response.status,
path: event.url.pathname
});
}
if (session && response.headers.get('content-type')?.startsWith('text/event-stream')) {
response = protectEventStream(response,session.tokenHash,permission,event.request.signal);
response = protectEventStream(
response,
session.tokenHash,
permission,
event.request.signal
);
}
const headers = new Headers(response.headers);
headers.set('Cache-Control','no-store');
headers.set('Vary',[headers.get('Vary'),'Cookie'].filter(Boolean).join(', '));
headers.set('Referrer-Policy','same-origin');
headers.set('X-Content-Type-Options','nosniff');
headers.set('X-Frame-Options','DENY');
return new Response(response.body,{status:response.status,statusText:response.statusText,headers});
headers.set('Cache-Control', 'no-store');
headers.set('Vary', [headers.get('Vary'), 'Cookie'].filter(Boolean).join(', '));
headers.set('Referrer-Policy', 'same-origin');
headers.set('X-Content-Type-Options', 'nosniff');
headers.set('X-Frame-Options', 'DENY');
return new Response(response.body, {
status: response.status,
statusText: response.statusText,
headers
});
};
+76 -22
View File
@@ -2,48 +2,102 @@ import { error, isHttpError, json, type Cookies } from '@sveltejs/kit';
import { AuthError, SESSION_COOKIE, type AuthResult } from './index.js';
import type { AuthUser } from '../../auth.js';
export function sessionCookie(url: URL): string { return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev'; }
export function sessionCookie(url: URL): string {
return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev';
}
export function requireUserTransport(url: URL): void {
if (url.protocol === 'https:') return;
if (process.env.EC_AUTH_ALLOW_HTTP === 'true' && ['localhost','127.0.0.1','[::1]','::1'].includes(url.hostname)) return;
error(426, 'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.');
if (
process.env.EC_AUTH_ALLOW_HTTP === 'true' &&
['localhost', '127.0.0.1', '[::1]', '::1'].includes(url.hostname)
)
return;
error(
426,
'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.'
);
}
export function requireSameOrigin(request: Request, url: URL): void {
if (request.headers.get('origin') !== url.origin || request.headers.get('sec-fetch-site') === 'cross-site') {
if (
request.headers.get('origin') !== url.origin ||
request.headers.get('sec-fetch-site') === 'cross-site'
) {
error(403, 'This action requires a same-origin request.');
}
}
export function sessionResponse(result: AuthResult, cookies: Cookies, url: URL) {
cookies.set(sessionCookie(url), result.session.token, { path: '/', httpOnly: true, secure: url.protocol === 'https:',
sameSite: 'lax', expires: new Date(result.session.expiresAt) });
return json({ user: result.user }, { headers: { 'Cache-Control':'no-store' } });
cookies.set(sessionCookie(url), result.session.token, {
path: '/',
httpOnly: true,
secure: url.protocol === 'https:',
sameSite: 'lax',
expires: new Date(result.session.expiresAt)
});
return json({ user: result.user }, { headers: { 'Cache-Control': 'no-store' } });
}
export function requireUser(user: AuthUser | null): AuthUser {
if (!user) throw new AuthError('Sign in to continue.',401);
if (!user) throw new AuthError('Sign in to continue.', 401);
return user;
}
export async function authBody(request: Request, maxBytes = 8192): Promise<Record<string, unknown>> {
if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json')) throw new AuthError('Send a JSON request.',415);
if (Number(request.headers.get('content-length')) > maxBytes) throw new AuthError('Request is too large.',413);
const reader = request.body?.getReader(); if (!reader) throw new AuthError('A JSON object is required.');
const chunks: Uint8Array[] = []; let length = 0;
export async function authBody(
request: Request,
maxBytes = 8192
): Promise<Record<string, unknown>> {
if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json'))
throw new AuthError('Send a JSON request.', 415);
if (Number(request.headers.get('content-length')) > maxBytes)
throw new AuthError('Request is too large.', 413);
const reader = request.body?.getReader();
if (!reader) throw new AuthError('A JSON object is required.');
const chunks: Uint8Array[] = [];
let length = 0;
try {
for (;;) {
const { done, value } = await reader.read(); if (done) break;
const { done, value } = await reader.read();
if (done) break;
length += value.byteLength;
if (length > maxBytes) { void reader.cancel().catch(() => {}); throw new AuthError('Request is too large.',413); }
if (length > maxBytes) {
void reader.cancel().catch(() => {});
throw new AuthError('Request is too large.', 413);
}
chunks.push(value);
}
} finally { reader.releaseLock(); }
} finally {
reader.releaseLock();
}
let body: unknown;
try { body = JSON.parse(Buffer.concat(chunks,length).toString('utf8')); } catch { throw new AuthError('Invalid JSON.'); }
if (!body || typeof body !== 'object' || Array.isArray(body)) throw new AuthError('A JSON object is required.');
try {
body = JSON.parse(Buffer.concat(chunks, length).toString('utf8'));
} catch {
throw new AuthError('Invalid JSON.');
}
if (!body || typeof body !== 'object' || Array.isArray(body))
throw new AuthError('A JSON object is required.');
return body as Record<string, unknown>;
}
export function authError(cause: unknown): Response {
if (isHttpError(cause)) return json({ error: cause.body.message }, { status:cause.status, headers:{'Cache-Control':'no-store'} });
if (isHttpError(cause))
return json(
{ error: cause.body.message },
{ status: cause.status, headers: { 'Cache-Control': 'no-store' } }
);
const message = cause instanceof Error ? cause.message : 'Authentication request failed.';
const status = cause instanceof AuthError ? cause.status : message.includes('busy') ? 429 : message.startsWith('Use a password') ? 400 : 500;
return json({ error: status === 500 ? 'The authentication request could not be completed.' : message },
{ status, headers:{'Cache-Control':'no-store',...(status===429?{'Retry-After':'900'}:{})} });
const status =
cause instanceof AuthError
? cause.status
: message.includes('busy')
? 429
: message.startsWith('Use a password')
? 400
: 500;
return json(
{ error: status === 500 ? 'The authentication request could not be completed.' : message },
{
status,
headers: {
'Cache-Control': 'no-store',
...(status === 429 ? { 'Retry-After': '900' } : {})
}
}
);
}
+342 -106
View File
@@ -8,179 +8,415 @@ export const SESSION_COOKIE = '__Host-ec_session';
export const SESSION_LIFETIME_MS = 12 * 60 * 60_000;
export const SESSION_IDLE_MS = 30 * 60_000;
const digest = (value: string) => createHash('sha256').update(value).digest('hex');
function database() { const handle = db(); ensureAuthSchema(handle); return handle; }
type UserRow = { id: string; username: string; display_name: string; password_hash: string; role: UserRole; disabled: number;
must_change_password: number; created_at: number; updated_at: number; last_login_at: number | null };
function database() {
const handle = db();
ensureAuthSchema(handle);
return handle;
}
type UserRow = {
id: string;
username: string;
display_name: string;
password_hash: string;
role: UserRole;
disabled: number;
must_change_password: number;
created_at: number;
updated_at: number;
last_login_at: number | null;
};
export class AuthError extends Error {
constructor(message: string, public status = 400) { super(message); }
constructor(
message: string,
public status = 400
) {
super(message);
}
}
function present(row: UserRow): AuthUser {
return { id: row.id, username: row.username, displayName: row.display_name, role: row.role, disabled: !!row.disabled,
mustChangePassword: !!row.must_change_password, createdAt: new Date(row.created_at).toISOString(),
updatedAt: new Date(row.updated_at).toISOString(), lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString() };
return {
id: row.id,
username: row.username,
displayName: row.display_name,
role: row.role,
disabled: !!row.disabled,
mustChangePassword: !!row.must_change_password,
createdAt: new Date(row.created_at).toISOString(),
updatedAt: new Date(row.updated_at).toISOString(),
lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString()
};
}
function userRow(id: string) {
return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined;
}
export function getUser(id: string): AuthUser | null {
const row = userRow(id);
return row ? present(row) : null;
}
export function listUsers(): AuthUser[] {
return (
database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[]
).map(present);
}
export function setupRequired(): boolean {
return !database().prepare('SELECT 1 FROM auth_users LIMIT 1').get();
}
function userRow(id: string) { return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined; }
export function getUser(id: string): AuthUser | null { const row = userRow(id); return row ? present(row) : null; }
export function listUsers(): AuthUser[] { return (database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[]).map(present); }
export function setupRequired(): boolean { return !(database().prepare('SELECT 1 FROM auth_users LIMIT 1').get()); }
function username(value: unknown): string {
if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value)) throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.');
if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value))
throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.');
const normalized = value.toLowerCase();
if (normalized === systemUsername.toLowerCase()) throw new AuthError(`${systemUsername} is reserved for system actions. Choose another username.`);
if (normalized === systemUsername.toLowerCase())
throw new AuthError(
`${systemUsername} is reserved for system actions. Choose another username.`
);
return normalized;
}
function displayName(value: unknown, fallback: string): string {
const name = value === undefined ? fallback : value;
if (typeof name !== 'string' || !name.trim() || name.length > 100 || /[\x00-\x1f\x7f]/.test(name)) throw new AuthError('Display name must contain 1–100 printable characters.');
if (
typeof name !== 'string' ||
!name.trim() ||
name.length > 100 ||
// eslint-disable-next-line no-control-regex -- Reject control characters in display names.
/[\x00-\x1f\x7f]/.test(name)
)
throw new AuthError('Display name must contain 1–100 printable characters.');
return name.trim();
}
function role(value: unknown): UserRole {
if (!['admin','operator','viewer'].includes(value as string)) throw new AuthError('Choose administrator, operator or viewer.');
if (!['admin', 'operator', 'viewer'].includes(value as string))
throw new AuthError('Choose administrator, operator or viewer.');
return value as UserRole;
}
export function recordAudit(actorId: string | null, action: string, targetId: string | null = null, details: Record<string, unknown> = {}): void {
database().prepare('INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)')
export function recordAudit(
actorId: string | null,
action: string,
targetId: string | null = null,
details: Record<string, unknown> = {}
): void {
database()
.prepare(
'INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)'
)
.run(actorId, action, targetId, JSON.stringify(details), Date.now());
}
export function listAuthAudit() {
return database().prepare(`SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt
FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200`).all();
return database()
.prepare(
`SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt
FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200`
)
.all();
}
/** Persistent fixed windows are consumed before hashing, including failed setup. */
export function consumeAuthAttempt(clientAddress: string, account: string, kind = 'login'): void {
const now = Date.now(); const windowMs = 15 * 60_000;
const now = Date.now();
const windowMs = 15 * 60_000;
const ipKey = digest(`${kind}:ip:${clientAddress}`);
const accountKey = digest(`${kind}:account:${account.toLowerCase()}`);
database().transaction(() => {
database().prepare('DELETE FROM auth_rate_limits WHERE window_start < ?').run(now - 24 * 60 * 60_000);
for (const [key, limit] of [[ipKey, 30], [accountKey, 10]] as const) {
const previous = database().prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?').get(key) as { window_start: number; attempts: number } | undefined;
if (previous && previous.window_start > now - windowMs && previous.attempts >= limit) throw new AuthError('Too many attempts. Try again in 15 minutes.', 429);
}
for (const key of [ipKey, accountKey]) database().prepare(`INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1)
database()
.transaction(() => {
database()
.prepare('DELETE FROM auth_rate_limits WHERE window_start < ?')
.run(now - 24 * 60 * 60_000);
for (const [key, limit] of [
[ipKey, 30],
[accountKey, 10]
] as const) {
const previous = database()
.prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?')
.get(key) as { window_start: number; attempts: number } | undefined;
if (
previous &&
previous.window_start > now - windowMs &&
previous.attempts >= limit
)
throw new AuthError('Too many attempts. Try again in 15 minutes.', 429);
}
for (const key of [ipKey, accountKey])
database()
.prepare(
`INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1)
ON CONFLICT(key) DO UPDATE SET attempts=CASE WHEN window_start <= ? THEN 1 ELSE attempts+1 END,
window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END`).run(key, now, now - windowMs, now - windowMs);
}).immediate();
window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END`
)
.run(key, now, now - windowMs, now - windowMs);
})
.immediate();
}
function insertUser(input: { username: string; displayName: string; role: UserRole; passwordHash: string; mustChangePassword: boolean }): AuthUser {
if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username)) throw new AuthError('That username is already in use.', 409);
const id = randomUUID(); const now = Date.now();
database().prepare(`INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`)
.run(id,input.username,input.displayName,input.passwordHash,input.role,Number(input.mustChangePassword),now,now);
function insertUser(input: {
username: string;
displayName: string;
role: UserRole;
passwordHash: string;
mustChangePassword: boolean;
}): AuthUser {
if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username))
throw new AuthError('That username is already in use.', 409);
const id = randomUUID();
const now = Date.now();
database()
.prepare(
`INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`
)
.run(
id,
input.username,
input.displayName,
input.passwordHash,
input.role,
Number(input.mustChangePassword),
now,
now
);
return getUser(id)!;
}
export interface AuthResult { user: AuthUser; session: { token: string; expiresAt: number } }
export async function bootstrapAdmin(input: Record<string, unknown>, clientAddress: string): Promise<AuthResult> {
export interface AuthResult {
user: AuthUser;
session: { token: string; expiresAt: number };
}
export async function bootstrapAdmin(
input: Record<string, unknown>,
clientAddress: string
): Promise<AuthResult> {
consumeAuthAttempt(clientAddress, 'bootstrap', 'setup');
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
const expected = process.env.EC_SETUP_TOKEN;
if (!expected || expected.length < 32) throw new AuthError('The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.', 503);
if (typeof input.token !== 'string' || !timingSafeEqual(Buffer.from(digest(input.token), 'hex'), Buffer.from(digest(expected), 'hex'))) throw new AuthError('Invalid setup token.', 401);
const name = username(input.username); const label = displayName(input.displayName, name);
if (!expected || expected.length < 32)
throw new AuthError(
'The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.',
503
);
if (
typeof input.token !== 'string' ||
!timingSafeEqual(
Buffer.from(digest(input.token), 'hex'),
Buffer.from(digest(expected), 'hex')
)
)
throw new AuthError('Invalid setup token.', 401);
const name = username(input.username);
const label = displayName(input.displayName, name);
const passwordHash = await hashPassword(input.password);
return database().transaction(() => {
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
const user = insertUser({ username: name, displayName: label, passwordHash, role: 'admin', mustChangePassword: false });
recordAudit(user.id, 'setup', user.id);
return { user, session: createSession(user.id) };
}).immediate();
return database()
.transaction(() => {
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
const user = insertUser({
username: name,
displayName: label,
passwordHash,
role: 'admin',
mustChangePassword: false
});
recordAudit(user.id, 'setup', user.id);
return { user, session: createSession(user.id) };
})
.immediate();
}
export async function authenticatePassword(input: Record<string, unknown>, clientAddress: string): Promise<AuthResult> {
const name = typeof input.username === 'string' ? input.username.slice(0,64).toLowerCase() : '';
export async function authenticatePassword(
input: Record<string, unknown>,
clientAddress: string
): Promise<AuthResult> {
const name =
typeof input.username === 'string' ? input.username.slice(0, 64).toLowerCase() : '';
consumeAuthAttempt(clientAddress, name);
const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as UserRow | undefined;
const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as
UserRow | undefined;
const valid = await verifyPassword(input.password, row?.password_hash ?? null);
const result = database().transaction(() => {
const current = row ? userRow(row.id) : undefined;
if (!valid || !current || current.disabled || current.password_hash !== row?.password_hash) {
recordAudit(null, 'login.failed'); return null;
}
database().prepare('UPDATE auth_users SET last_login_at=? WHERE id=?').run(Date.now(), current.id);
recordAudit(current.id, 'login', current.id);
return { user: getUser(current.id)!, session: createSession(current.id) };
}).immediate();
const result = database()
.transaction(() => {
const current = row ? userRow(row.id) : undefined;
if (
!valid ||
!current ||
current.disabled ||
current.password_hash !== row?.password_hash
) {
recordAudit(null, 'login.failed');
return null;
}
database()
.prepare('UPDATE auth_users SET last_login_at=? WHERE id=?')
.run(Date.now(), current.id);
recordAudit(current.id, 'login', current.id);
return { user: getUser(current.id)!, session: createSession(current.id) };
})
.immediate();
if (!result) throw new AuthError('Invalid username or password.', 401);
return result;
}
export function createSession(userId: string): { token: string; expiresAt: number } {
const now = Date.now(); const user = userRow(userId);
const now = Date.now();
const user = userRow(userId);
if (!user || user.disabled) throw new AuthError('Account is unavailable.', 401);
const token = randomBytes(32).toString('base64url'); const expiresAt = now + SESSION_LIFETIME_MS;
database().prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?').run(now,now-SESSION_IDLE_MS);
database().prepare('INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)').run(digest(token),userId,now,expiresAt,now);
const token = randomBytes(32).toString('base64url');
const expiresAt = now + SESSION_LIFETIME_MS;
database()
.prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?')
.run(now, now - SESSION_IDLE_MS);
database()
.prepare(
'INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)'
)
.run(digest(token), userId, now, expiresAt, now);
return { token, expiresAt };
}
export function readSession(token: string | undefined, touch = true): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
export function readSession(
token: string | undefined,
touch = true
): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
if (!token || !/^[A-Za-z0-9_-]{43}$/.test(token)) return null;
return readSessionHash(digest(token), touch);
}
export function readSessionHash(tokenHash: string, touch = false): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
export function readSessionHash(
tokenHash: string,
touch = false
): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
const now = Date.now();
const row = database().prepare(`SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id
WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0`).get(tokenHash,now,now-SESSION_IDLE_MS) as (UserRow & { expires_at: number; last_seen_at: number }) | undefined;
const row = database()
.prepare(
`SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id
WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0`
)
.get(tokenHash, now, now - SESSION_IDLE_MS) as
(UserRow & { expires_at: number; last_seen_at: number }) | undefined;
if (!row) return null;
if (touch && row.last_seen_at < now-60_000) database().prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?').run(now,tokenHash);
if (touch && row.last_seen_at < now - 60_000)
database()
.prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?')
.run(now, tokenHash);
return { user: present(row), tokenHash, expiresAt: row.expires_at };
}
export function revokeSession(token: string | undefined): void { if (token) database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token)); }
export function revokeSession(token: string | undefined): void {
if (token)
database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token));
}
export function revokeUserSessions(actorId: string, userId: string): void {
database().transaction(() => {
requireAdministrator(actorId);
if (!userRow(userId)) throw new AuthError('User not found.',404);
if (!userRow(userId)) throw new AuthError('User not found.', 404);
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
recordAudit(actorId,'sessions.revoked',userId);
recordAudit(actorId, 'sessions.revoked', userId);
})();
}
function requireAdministrator(actorId: string) {
const actor = userRow(actorId);
if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password) throw new AuthError('Administrator access is required.',403);
if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password)
throw new AuthError('Administrator access is required.', 403);
}
export async function createUser(actorId: string, input: Record<string, unknown>): Promise<AuthUser> {
export async function createUser(
actorId: string,
input: Record<string, unknown>
): Promise<AuthUser> {
requireAdministrator(actorId);
const name = username(input.username), label = displayName(input.displayName,name), userRole = role(input.role);
const name = username(input.username),
label = displayName(input.displayName, name),
userRole = role(input.role);
const passwordHash = await hashPassword(input.password);
return database().transaction(() => {
requireAdministrator(actorId);
const user = insertUser({ username:name,displayName:label,role:userRole,passwordHash,mustChangePassword:true });
recordAudit(actorId,'user.created',user.id,{role:userRole}); return user;
}).immediate();
return database()
.transaction(() => {
requireAdministrator(actorId);
const user = insertUser({
username: name,
displayName: label,
role: userRole,
passwordHash,
mustChangePassword: true
});
recordAudit(actorId, 'user.created', user.id, { role: userRole });
return user;
})
.immediate();
}
export async function updateUser(actorId: string, id: string, input: Record<string, unknown>): Promise<AuthUser> {
export async function updateUser(
actorId: string,
id: string,
input: Record<string, unknown>
): Promise<AuthUser> {
requireAdministrator(actorId);
if (input.disabled !== undefined && typeof input.disabled !== 'boolean') throw new AuthError('Disabled must be true or false.');
if (input.password !== undefined && actorId === id) throw new AuthError('Change your own password from Account using your current password.');
if (input.disabled !== undefined && typeof input.disabled !== 'boolean')
throw new AuthError('Disabled must be true or false.');
if (input.password !== undefined && actorId === id)
throw new AuthError('Change your own password from Account using your current password.');
const passwordHash = input.password === undefined ? null : await hashPassword(input.password);
return database().transaction(() => {
requireAdministrator(actorId);
const current = userRow(id); if (!current) throw new AuthError('User not found.',404);
const nextRole = input.role === undefined ? current.role : role(input.role);
const disabled = input.disabled === undefined ? current.disabled : Number(input.disabled);
const label = displayName(input.displayName,current.display_name);
if (current.role === 'admin' && !current.disabled && (nextRole !== 'admin' || disabled)) {
const other = database().prepare("SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1").get(id);
if (!other) throw new AuthError('Keep at least one enabled administrator.',409);
}
database().prepare(`UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?`)
.run(label,nextRole,disabled,passwordHash??current.password_hash,passwordHash?1:current.must_change_password,Date.now(),id);
if (passwordHash || disabled !== current.disabled || nextRole !== current.role) database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id);
recordAudit(actorId,'user.updated',id,{role:nextRole,disabled:!!disabled,passwordReset:!!passwordHash});
return getUser(id)!;
}).immediate();
return database()
.transaction(() => {
requireAdministrator(actorId);
const current = userRow(id);
if (!current) throw new AuthError('User not found.', 404);
const nextRole = input.role === undefined ? current.role : role(input.role);
const disabled =
input.disabled === undefined ? current.disabled : Number(input.disabled);
const label = displayName(input.displayName, current.display_name);
if (
current.role === 'admin' &&
!current.disabled &&
(nextRole !== 'admin' || disabled)
) {
const other = database()
.prepare(
"SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1"
)
.get(id);
if (!other) throw new AuthError('Keep at least one enabled administrator.', 409);
}
database()
.prepare(
`UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?`
)
.run(
label,
nextRole,
disabled,
passwordHash ?? current.password_hash,
passwordHash ? 1 : current.must_change_password,
Date.now(),
id
);
if (passwordHash || disabled !== current.disabled || nextRole !== current.role)
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id);
recordAudit(actorId, 'user.updated', id, {
role: nextRole,
disabled: !!disabled,
passwordReset: !!passwordHash
});
return getUser(id)!;
})
.immediate();
}
export async function changePassword(userId: string, currentPassword: unknown, newPassword: unknown): Promise<AuthResult> {
export async function changePassword(
userId: string,
currentPassword: unknown,
newPassword: unknown
): Promise<AuthResult> {
const original = userRow(userId);
if (!original || original.disabled || !await verifyPassword(currentPassword,original.password_hash)) throw new AuthError('Current password is incorrect.',400);
if (
!original ||
original.disabled ||
!(await verifyPassword(currentPassword, original.password_hash))
)
throw new AuthError('Current password is incorrect.', 400);
if (currentPassword === newPassword) throw new AuthError('Choose a different password.');
const passwordHash = await hashPassword(newPassword);
return database().transaction(() => {
const current = userRow(userId);
if (!current || current.disabled || current.password_hash !== original.password_hash) throw new AuthError('Account changed during password verification. Sign in again.',409);
database().prepare('UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?').run(passwordHash,Date.now(),userId);
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
recordAudit(userId,'password.changed',userId);
return { user: getUser(userId)!, session: createSession(userId) };
}).immediate();
return database()
.transaction(() => {
const current = userRow(userId);
if (!current || current.disabled || current.password_hash !== original.password_hash)
throw new AuthError(
'Account changed during password verification. Sign in again.',
409
);
database()
.prepare(
'UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?'
)
.run(passwordHash, Date.now(), userId);
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
recordAudit(userId, 'password.changed', userId);
return { user: getUser(userId)!, session: createSession(userId) };
})
.immediate();
}
+15 -5
View File
@@ -15,12 +15,14 @@ export function validatePassword(password) {
/** @param {string} password @param {string} salt @returns {Promise<Buffer>} */
function derive(password, salt) {
if (active >= 2) return Promise.reject(new Error('Password service is busy. Try again shortly.'));
if (active >= 2)
return Promise.reject(new Error('Password service is busy. Try again shortly.'));
active++;
return new Promise((resolve, reject) => {
scrypt(password, salt, 64, parameters, (error, result) => {
active--;
if (error) reject(error); else resolve(result);
if (error) reject(error);
else resolve(result);
});
});
}
@@ -37,9 +39,17 @@ export async function hashPassword(password) {
export async function verifyPassword(password, encoded) {
if (typeof password !== 'string' || [...password].length > 128) return false;
const parts = encoded?.split('$');
const valid = parts?.length === 6 && parts[0] === 'scrypt' && parts[1] === '131072' && parts[2] === '8' && parts[3] === '1'
&& /^[a-f0-9]{64}$/.test(parts[4]) && /^[a-f0-9]{128}$/.test(parts[5]);
const valid =
parts?.length === 6 &&
parts[0] === 'scrypt' &&
parts[1] === '131072' &&
parts[2] === '8' &&
parts[3] === '1' &&
/^[a-f0-9]{64}$/.test(parts[4]) &&
/^[a-f0-9]{128}$/.test(parts[5]);
// Missing accounts perform the same expensive operation as real accounts.
const result = await derive(password, valid ? parts[4] : '0'.repeat(64));
return timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid;
return (
timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid
);
}
+9 -3
View File
@@ -3,11 +3,17 @@ import { AsyncLocalStorage } from 'node:async_hooks';
const requester = new AsyncLocalStorage<{ username: string | null; active: boolean }>();
/** Use only the authenticated session, and stop attribution when the request finishes. */
export async function withRequestUsername<T>(username: string | null, work: () => T | Promise<T>): Promise<T> {
export async function withRequestUsername<T>(
username: string | null,
work: () => T | Promise<T>
): Promise<T> {
const context = { username, active: true };
return requester.run(context, async () => {
try { return await work(); }
finally { context.active = false; }
try {
return await work();
} finally {
context.active = false;
}
});
}
+17 -8
View File
@@ -1,24 +1,32 @@
import { createHash, timingSafeEqual } from 'node:crypto';
export type TelemetryAuthentication =
| { ok: true }
| { ok: false; status: 401 | 503; error: string };
{ ok: true } | { ok: false; status: 401 | 503; error: string };
const tokenPattern = /^[A-Za-z0-9._~+\/-]+=*$/;
const validToken = (value: string): boolean => value.length >= 32 && value.length <= 512 && tokenPattern.test(value);
const tokenPattern = /^[A-Za-z0-9._~+/-]+=*$/;
const validToken = (value: string): boolean =>
value.length >= 32 && value.length <= 512 && tokenPattern.test(value);
const hash = (value: string): Buffer => createHash('sha256').update(value, 'utf8').digest();
/** Machine telemetry has its own credential; browser sessions never authorize it. */
export function authenticateTelemetry(request: Request): TelemetryAuthentication {
const expected = process.env.EC_TELEMETRY_TOKEN ?? '';
if (!validToken(expected)) {
return { ok: false, status: 503, error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.' };
return {
ok: false,
status: 503,
error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.'
};
}
const authorization = request.headers.get('authorization');
const custom = request.headers.get('x-ec-telemetry-token');
const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+\/-]+=*)$/i)?.[1];
const denied = (): TelemetryAuthentication => ({ ok: false, status: 401, error: 'A valid game telemetry token is required.' });
const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+/-]+=*)$/i)?.[1];
const denied = (): TelemetryAuthentication => ({
ok: false,
status: 401,
error: 'A valid game telemetry token is required.'
});
// Reject malformed/ambiguous credentials rather than falling back to a
// second header. A shared secret must never come from a URL, cookie or body.
@@ -28,7 +36,8 @@ export function authenticateTelemetry(request: Request): TelemetryAuthentication
const supplied = bearer ?? custom;
if (!supplied) return denied();
const suppliedHash = hash(supplied);
if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom))) return denied();
if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom)))
return denied();
if (!timingSafeEqual(suppliedHash, hash(expected))) return denied();
return { ok: true };
}
+55 -10
View File
@@ -4,29 +4,74 @@ import { scalingSnapshot } from './snapshot.js';
import { liveScaling, type ScalingPolicy, type ScalingDecision } from '../../autoscaling.js';
import { deploymentStage } from '../provisioning/history.js';
export interface ScalingIntent { policy_id: string; revision: number; decision: string; operation_id: string }
export interface ScalingIntent {
policy_id: string;
revision: number;
decision: string;
operation_id: string;
}
export function scalingIntent(operationId: string): ScalingIntent | undefined {
return db().prepare('SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0').get(operationId) as ScalingIntent | undefined;
return db()
.prepare(
'SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0'
)
.get(operationId) as ScalingIntent | undefined;
}
/** Rechecked before placement, after asynchronous preparation, and before dispatch. */
export function scalingAdmission(operationId: string): string | null {
const intent = scalingIntent(operationId);
if (!intent) return null;
const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(intent.policy_id) as { id: string; settings: string; mode: ScalingPolicy['mode']; revision: number; paused_reason: string | null; owns_map: number } | undefined;
const row = db()
.prepare('SELECT * FROM autoscaling_policies WHERE id=?')
.get(intent.policy_id) as
| {
id: string;
settings: string;
mode: ScalingPolicy['mode'];
revision: number;
paused_reason: string | null;
owns_map: number;
}
| undefined;
let reason: string | null = null;
const paused = (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
if (!row || !liveScaling(row.mode) || !row.owns_map || row.revision !== intent.revision || row.paused_reason || paused) reason = 'Autoscaling policy is paused or changed.';
const paused =
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1';
if (
!row ||
!liveScaling(row.mode) ||
!row.owns_map ||
row.revision !== intent.revision ||
row.paused_reason ||
paused
)
reason = 'Autoscaling policy is paused or changed.';
else {
const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, pausedReason: null, updatedAt: '' };
const policy: ScalingPolicy = {
...JSON.parse(row.settings),
id: row.id,
mode: row.mode,
revision: row.revision,
pausedReason: null,
updatedAt: ''
};
const requested = JSON.parse(intent.decision) as ScalingDecision;
const snapshot = scalingSnapshot(policy, Date.now(), operationId);
if (requested.action === 'create') {
snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server);
snapshot.blockers = snapshot.blockers.filter(b => b !== `Unresolved native instance ${requested.server}.`);
snapshot.reservedNames = snapshot.reservedNames.filter((n) => n !== requested.server);
snapshot.blockers = snapshot.blockers.filter(
(b) => b !== `Unresolved native instance ${requested.server}.`
);
}
const current = planScaling(policy, snapshot, Date.now());
if (current.action !== requested.action || current.server !== requested.server) reason = `Autoscaling intent no longer eligible: ${current.reason}`;
if (current.action !== requested.action || current.server !== requested.server)
reason = `Autoscaling intent no longer eligible: ${current.reason}`;
}
if (reason) deploymentStage(operationId, reason, { blocker: { code: 'autoscaling', message: reason, automatic: true } });
if (reason)
deploymentStage(operationId, reason, {
blocker: { code: 'autoscaling', message: reason, automatic: true }
});
return reason;
}
+77 -17
View File
@@ -7,46 +7,106 @@ export const autoscalingRemovalCapability = 'instance.autoscaling-remove';
export function removalAuthorization(operationId: string, hostId: string): object | null {
const intent = scalingIntent(operationId);
if (!intent) return null;
if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) throw new Error('Update the host agent before automatic removal.');
if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability))
throw new Error('Update the host agent before automatic removal.');
const reason = scalingAdmission(operationId);
if (reason) throw new Error(reason);
const expiresAt = Date.now() + 60_000;
db().prepare('INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)')
db()
.prepare(
'INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)'
)
.run(operationId, intent.policy_id, intent.revision, expiresAt);
return { policyId: intent.policy_id, revision: intent.revision, expiresAt };
}
/** Claim is the cancellation boundary. Lost replies never authorize a second claim. */
export function claimRemovalAuthorization(hostId: string, commandId: string, input: Record<string, unknown>): { authorized: boolean; validForMs?: number; reason?: string } {
export function claimRemovalAuthorization(
hostId: string,
commandId: string,
input: Record<string, unknown>
): { authorized: boolean; validForMs?: number; reason?: string } {
return db().transaction(() => {
const now = Date.now();
const row = db().prepare(`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
const row = db()
.prepare(
`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
JOIN native_provisioning n ON n.deployment_id=a.operation_id JOIN server_deployments d ON d.id=n.deployment_id
JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
JOIN ec_host_operations o ON o.id=n.operation_id AND o.host_id=i.host_id AND o.instance_id=i.id AND o.generation=i.generation
WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`).get(commandId, hostId) as
{ operation_id: string; policy_id: string; revision: number; expires_at: number; claimed_at: number | null; revoked_at: number | null; deployment_state: string; command_state: string } | undefined;
if (!row || input.policyId !== row.policy_id || input.revision !== row.revision || input.expiresAt !== row.expires_at
|| row.claimed_at !== null || row.revoked_at !== null || row.expires_at <= now
|| row.deployment_state !== 'running' || !['pending','accepted','running'].includes(row.command_state)) return { authorized: false, reason: 'Removal authorization expired, changed, or was already claimed.' };
WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`
)
.get(commandId, hostId) as
| {
operation_id: string;
policy_id: string;
revision: number;
expires_at: number;
claimed_at: number | null;
revoked_at: number | null;
deployment_state: string;
command_state: string;
}
| undefined;
if (
!row ||
input.policyId !== row.policy_id ||
input.revision !== row.revision ||
input.expiresAt !== row.expires_at ||
row.claimed_at !== null ||
row.revoked_at !== null ||
row.expires_at <= now ||
row.deployment_state !== 'running' ||
!['pending', 'accepted', 'running'].includes(row.command_state)
)
return {
authorized: false,
reason: 'Removal authorization expired, changed, or was already claimed.'
};
const reason = scalingAdmission(row.operation_id);
if (reason) { db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?').run(now, row.operation_id); return { authorized: false, reason }; }
db().prepare('UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL').run(now, row.operation_id);
if (reason) {
db()
.prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?')
.run(now, row.operation_id);
return { authorized: false, reason };
}
db()
.prepare(
'UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL'
)
.run(now, row.operation_id);
// Only a short final occupancy check may occur after this boundary.
return { authorized: true, validForMs: Math.min(5000, row.expires_at - now) };
})();
}
export function revokePolicyRemovals(policyId: string): void {
if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId)) return;
db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL').run(Date.now(), policyId);
if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId))
return;
db()
.prepare(
'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL'
)
.run(Date.now(), policyId);
const at = new Date().toISOString();
const commands = db().prepare(`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
const commands = db()
.prepare(
`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
JOIN autoscaling_authorizations a ON a.operation_id=n.deployment_id
WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`).all(policyId) as { id: string }[];
WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`
)
.all(policyId) as { id: string }[];
for (const command of commands) {
const result = JSON.stringify({ autoscalingInvalidated: true });
db().prepare("UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?").run(result, at, command.id);
db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')").run(command.id, at);
db()
.prepare(
"UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?"
)
.run(result, at, command.id);
db()
.prepare(
"INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')"
)
.run(command.id, at);
}
}
+428 -87
View File
@@ -1,6 +1,13 @@
import { createHash, randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingEvaluation, type ScalingReport, type ScalingView } from '../../autoscaling.js';
import {
liveScaling,
scalingSettings,
type ScalingPolicy,
type ScalingEvaluation,
type ScalingReport,
type ScalingView
} from '../../autoscaling.js';
import { emptyObservation, planScaling } from './planner.js';
import { parseServerName } from '../../map-aliases.js';
import { scalingSnapshot } from './snapshot.js';
@@ -14,33 +21,98 @@ import { getHost } from '../hosts/index.js';
import { scalingOwner } from './ownership.js';
import { getServer } from '../fleet/servers.js';
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; paused_reason: string | null; updated_at: string }
const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, pausedReason: row.paused_reason, updatedAt: row.updated_at });
export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); }
export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
export class ScalingError extends Error {
constructor(
message: string,
public status = 400
) {
super(message);
}
}
interface PolicyRow {
id: string;
revision: number;
settings: string;
mode: ScalingPolicy['mode'];
paused_reason: string | null;
updated_at: string;
}
const present = (row: PolicyRow): ScalingPolicy => ({
...JSON.parse(row.settings),
id: row.id,
revision: row.revision,
mode: row.mode,
pausedReason: row.paused_reason,
updatedAt: row.updated_at
});
export const listScalingPolicies = (): ScalingPolicy[] =>
(
db()
.prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug')
.all() as PolicyRow[]
).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined {
const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as
PolicyRow | undefined;
return row && present(row);
}
export const scalingGloballyPaused = (): boolean =>
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1';
export function evaluateScaling(input: unknown, id?: string, revision?: number): ScalingEvaluation {
const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined;
const settings = scalingSettings(input),
existing = id ? getScalingPolicy(id) : undefined;
if (id && !existing) throw new ScalingError('Policy not found.', 404);
if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'new-policy', revision: existing?.revision ?? 0,
pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
if (existing && existing.revision !== revision)
throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug))
throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = {
...settings,
id: existing?.id ?? 'new-policy',
revision: existing?.revision ?? 0,
pausedReason: null,
updatedAt: existing?.updatedAt ?? ''
};
const snapshot = scalingSnapshot(policy);
const blockers: string[] = [];
if (liveScaling(policy.mode)) {
const owner = scalingOwner(policy.game, policy.mapSlug);
if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`);
if (owner && owner !== policy.id)
blockers.push(
`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`
);
blockers.push(...snapshot.blockers);
for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`);
if (policy.mode === 'create-destroy') for (const member of snapshot.members) {
const hostId = getServer(member.name)?.host_id;
if (!hostId || !getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) blockers.push(`Update the host agent for ${member.name} before enabling automatic removal.`);
for (const member of snapshot.members)
if (!['occupied', 'empty'].includes(member.state))
blockers.push(
`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`
);
if (policy.mode === 'create-destroy')
for (const member of snapshot.members) {
const hostId = getServer(member.name)?.host_id;
if (
!hostId ||
!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)
)
blockers.push(
`Update the host agent for ${member.name} before enabling automatic removal.`
);
}
if (
db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(policy.id)
)
blockers.push('Settle the current autoscaling operation before editing a live policy.');
try {
nativeLaunchSelection({ automatic: true }, policy.game);
} catch (error) {
blockers.push((error as Error).message);
}
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.');
try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); }
}
const decision = planScaling(policy, snapshot, Date.now());
return { policy, snapshot, decision, blockers };
@@ -50,16 +122,48 @@ export function saveScaling(input: unknown, id?: string, revision?: number): Sca
return db().transaction(() => {
const evaluation = evaluateScaling(input, id, revision);
if (evaluation.blockers.length) throw new ScalingError(evaluation.blockers[0]);
const policy = evaluation.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
const policy = evaluation.policy,
settings = scalingSettings(policy),
nextId = id ?? randomUUID(),
at = new Date().toISOString();
if (id) revokePolicyRemovals(id);
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this scope.', 409);
db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
if (
!id &&
db()
.prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?')
.get(policy.game, policy.mapSlug)
)
throw new ScalingError('A policy already exists for this scope.', 409);
db()
.prepare(
`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`
)
.run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
// An edit starts a new observation interval; earlier inactivity is not authorization.
db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId);
db().prepare('UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?').run(liveScaling(policy.mode) ? 1 : 0, nextId);
if (liveScaling(policy.mode)) for (const member of evaluation.snapshot.members) db().prepare('INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)')
.run(nextId, member.name, member.identity, JSON.stringify({ identity: member.identity, observedAt: member.observedAt, emptySince: null, evaluatedAt: Date.now() }));
db()
.prepare(
'UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?'
)
.run(liveScaling(policy.mode) ? 1 : 0, nextId);
if (liveScaling(policy.mode))
for (const member of evaluation.snapshot.members)
db()
.prepare(
'INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)'
)
.run(
nextId,
member.name,
member.identity,
JSON.stringify({
identity: member.identity,
observedAt: member.observedAt,
emptySince: null,
evaluatedAt: Date.now()
})
);
reconcileScaling(getScalingPolicy(nextId)!);
return getScalingPolicy(nextId)!;
})();
@@ -69,10 +173,18 @@ export function deleteScaling(id: string, revision: number): void {
db().transaction(() => {
const policy = getScalingPolicy(id);
if (!policy) throw new ScalingError('Policy not found.', 404);
if (policy.revision !== revision) throw new ScalingError('Policy changed. Refresh before deleting.', 409);
if (policy.revision !== revision)
throw new ScalingError('Policy changed. Refresh before deleting.', 409);
// Keep admission evidence and ownership while a host command may still act.
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(id)) {
throw new ScalingError('Pause this policy and settle its outstanding Provisioning work before deleting it.', 409);
if (
db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(id)
) {
throw new ScalingError(
'Pause this policy and settle its outstanding Provisioning work before deleting it.',
409
);
}
revokePolicyRemovals(id);
db().prepare('DELETE FROM autoscaling_authorizations WHERE policy_id=?').run(id);
@@ -85,39 +197,105 @@ export function deleteScaling(id: string, revision: number): void {
export function setScalingGlobalPause(paused: boolean): void {
db().transaction(() => {
db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
if (paused) for (const policy of listScalingPolicies()) { revokePolicyRemovals(policy.id); reconcileScaling(policy); }
db()
.prepare(
"INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value"
)
.run(paused ? '1' : '0');
if (paused)
for (const policy of listScalingPolicies()) {
revokePolicyRemovals(policy.id);
reconcileScaling(policy);
}
})();
}
function reconcileScaling(policy: ScalingPolicy): void {
const intents = db().prepare(`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
const intents = db()
.prepare(
`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as
{ id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[];
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`
)
.all(policy.id) as {
id: string;
revision: number;
operation_id: string;
action: string;
state: string;
submitted_at: string | null;
command_id: string | null;
}[];
for (const intent of intents) {
if (intent.action === 'create' && ['queued','waiting','cancelled'].includes(intent.state) && !intent.command_id && !intent.submitted_at
&& (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
if (
intent.action === 'create' &&
['queued', 'waiting', 'cancelled'].includes(intent.state) &&
!intent.command_id &&
!intent.submitted_at &&
(scalingAdmission(intent.operation_id) || intent.state === 'cancelled')
)
cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
if (intent.action === 'destroy' && scalingAdmission(intent.operation_id)) {
revokePolicyRemovals(policy.id);
if (!intent.command_id) cancelRemoval(intent.operation_id, false);
}
const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null };
const terminal = ['done','warning','failed','cancelled'].includes(current.state);
const command = intent.command_id ? db().prepare('SELECT state,result FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string; result: string | null } | undefined : undefined;
const invalidated = command?.state === 'failed' && command.result && JSON.parse(command.result).autoscalingInvalidated === true;
const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed'
&& !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id);
const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending;
if (!invalidated && (current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) {
db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id);
const current = db()
.prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?')
.get(intent.operation_id) as { state: string; server_deleted_at: string | null };
const terminal = ['done', 'warning', 'failed', 'cancelled'].includes(current.state);
const command = intent.command_id
? (db()
.prepare('SELECT state,result FROM ec_host_operations WHERE id=?')
.get(intent.command_id) as { state: string; result: string | null } | undefined)
: undefined;
const invalidated =
command?.state === 'failed' &&
command.result &&
JSON.parse(command.result).autoscalingInvalidated === true;
const receiptPending =
current.state === 'cancelled' &&
!current.server_deleted_at &&
command?.state === 'completed' &&
!!db()
.prepare(
'SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL'
)
.get(intent.operation_id);
const settled =
(!intent.command_id ||
(!!command && ['completed', 'failed'].includes(command.state))) &&
!receiptPending;
if (
!invalidated &&
(current.state === 'failed' ||
current.state === 'interrupted' ||
command?.state === 'unknown') &&
liveScaling(policy.mode)
) {
db()
.prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?')
.run(
`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`,
new Date().toISOString(),
policy.id
);
}
if (terminal && settled) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id);
if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled')) db().prepare("UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?").run(policy.id);
if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled'))
db()
.prepare(
"UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?"
)
.run(policy.id);
}
}
if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) {
if (
!liveScaling(policy.mode) &&
!db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(policy.id)
) {
db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id);
}
}
@@ -131,48 +309,152 @@ export async function runAutoscaling(): Promise<boolean> {
if (policy.mode === 'off') continue;
db().transaction(() => {
const snapshot = scalingSnapshot(policy, now);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined;
if (scalingGloballyPaused())
snapshot.blockers.unshift('Autoscaling is globally paused.');
const pending = db()
.prepare(
`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`
)
.get(policy.id) as { id: string; stage: string } | undefined;
if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`);
const decision = planScaling(policy, snapshot, now);
for (const member of snapshot.members) {
// Keep the observation interval that justified a pending removal;
// admission reads fresh evidence and revokes it on any new blocker.
if (member.state === 'removing' && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0').get(policy.id, member.operationId)) continue;
const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now);
db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation));
if (
member.state === 'removing' &&
db()
.prepare(
'SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0'
)
.get(policy.id, member.operationId)
)
continue;
const observation = emptyObservation(
member,
{
identity: member.identity,
observedAt: member.observedAt,
emptySince: member.emptySince,
evaluatedAt: now
},
now
);
db()
.prepare(
`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`
)
.run(policy.id, member.name, member.identity, JSON.stringify(observation));
}
// Evidence includes provenance and timestamps but never keys or player rosters.
const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyPool: snapshot.keyPool, blockers: snapshot.blockers });
const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
const execute = liveScaling(policy.mode) && (decision.action === 'create' || decision.action === 'destroy');
const evidence = JSON.stringify({
members: snapshot.members,
reservations: snapshot.reservedNames,
keyPool: snapshot.keyPool,
blockers: snapshot.blockers
});
const fingerprint = createHash('sha256')
.update(
JSON.stringify([
policy.revision,
decision,
snapshot.members.map((m) => [m.name, m.identity, m.state])
])
)
.digest('hex');
const last = db()
.prepare(
'SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1'
)
.get(policy.id) as { id: string; fingerprint: string } | undefined;
const execute =
liveScaling(policy.mode) &&
(decision.action === 'create' || decision.action === 'destroy');
const decisionId = randomUUID();
if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)')
.run(decisionId, policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence);
if (!execute && last?.fingerprint === fingerprint)
db()
.prepare(
'UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?'
)
.run(now, evidence, last.id);
else
db()
.prepare(
'INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)'
)
.run(
decisionId,
policy.id,
policy.revision,
now,
now,
fingerprint,
JSON.stringify(decision),
evidence
);
if (execute) {
db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
db()
.prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?')
.run(decisionId);
try {
db().transaction(() => {
const result = decision.action === 'create'
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
const result =
decision.action === 'create'
? queueNativeCreationIntent({
server: decision.server!,
game: policy.game,
mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection(
{ automatic: true },
policy.game
),
createdBy: `autoscaling:${policy.id}`,
autoscalingDecisionId: decisionId
})
: destroyServer(
decision.server!,
`autoscaling:${policy.id}`,
decisionId
);
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
if (!result.deploymentId)
throw new Error('Provisioning did not return an operation identity.');
db()
.prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?')
.run(result.deploymentId, decisionId);
db()
.prepare(
"INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)"
)
.run(result.deploymentId, policy.id, decision.server);
submitted = true;
})();
} catch (error) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Scaling action could not be queued: ${(error as Error).message}`, policy.id);
db()
.prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?')
.run(decisionId);
db()
.prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?')
.run(
`Scaling action could not be queued: ${(error as Error).message}`,
policy.id
);
}
}
db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, decision.deficit);
db()
.prepare(
`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`
)
.run(
policy.id,
Math.floor(now / 15_000) * 15_000,
policy.mode,
decision.occupied,
decision.empty,
decision.unknown,
decision.deficit
);
})();
}
return submitted;
@@ -180,37 +462,96 @@ export async function runAutoscaling(): Promise<boolean> {
export function pruneAutoscaling(now = Date.now()): void {
const cutoff = now - 31 * 86400_000;
db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`).run(cutoff);
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1').run(cutoff);
db()
.prepare(
`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`
)
.run(cutoff);
db()
.prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1')
.run(cutoff);
db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
}
export function scalingReport(): ScalingReport {
const policies: ScalingView[] = listScalingPolicies().map(policy => {
const policies: ScalingView[] = listScalingPolicies().map((policy) => {
const snapshot = scalingSnapshot(policy);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const decision = planScaling(policy, snapshot, Date.now());
const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
const history = (
db()
.prepare(
`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[])
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state }));
const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled'];
const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map;
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`
)
.all(policy.id) as {
id: string;
evaluated_at: number;
last_seen_at: number;
decision: string;
operation_id: string | null;
operation_state: string | null;
}[]
).map((row) => ({
id: row.id,
at: row.evaluated_at,
lastSeenAt: row.last_seen_at,
decision: JSON.parse(row.decision),
operationId: row.operation_id,
operationState: row.operation_state
}));
const unsettled = db()
.prepare(
`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`
)
.all(policy.id) as ScalingView['unsettled'];
const ownsMap = !!(
db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as {
owns_map: number;
}
).owns_map;
// Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
const totals = db()
.prepare(
`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`
)
.get(policy.id) as ScalingView['totals'];
const operations = db()
.prepare(
`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed,
COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples,
AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds,
MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds
FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations'];
return { policy, snapshot, decision, blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap };
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`
)
.get(
policy.id,
new Date(Date.now() - 31 * 86400_000).toISOString()
) as ScalingView['operations'];
return {
policy,
snapshot,
decision,
blockers: [],
history,
lastEvaluatedAt: history[0]?.lastSeenAt ?? null,
totals,
operations,
unsettled,
ownsMap
};
});
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true };
return {
policies,
globalPaused: scalingGloballyPaused(),
liveAvailable: true,
removalAvailable: true
};
}
+36 -8
View File
@@ -4,18 +4,29 @@ import { liveScaling, type ScalingPolicy } from '../../autoscaling.js';
import { mapIdentity } from './snapshot.js';
function owningPolicy(game: string, mapSlug: string) {
return db().prepare("SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1")
.get(game, mapSlug, mapSlug) as { id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined;
return db()
.prepare(
"SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1"
)
.get(game, mapSlug, mapSlug) as
{ id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined;
}
/** Durable ownership prevents overlapping policies even while scaling is paused. */
export function scalingOwner(game: string, mapSlug: string): string | null {
return owningPolicy(game, mapSlug)?.id ?? null;
}
const ownershipMessage = (owner: string) => `Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`;
const ownershipMessage = (owner: string) =>
`Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`;
function manualLineupBlocker(game: string, mapSlug: string): string | null {
const owner = owningPolicy(game, mapSlug);
if (!owner || !liveScaling(owner.mode) || owner.paused_reason) return null;
if ((db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1') return null;
if (
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1'
)
return null;
return ownershipMessage(owner.id);
}
export function scalingOwnershipBlocker(server: string): string | null {
@@ -30,13 +41,30 @@ export function assertScalingOwnership(game: string, mapSlug: string, decisionId
}
const owner = scalingOwner(game, mapSlug);
if (!owner) throw new Error('Autoscaling no longer owns this map.');
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return;
if (
db()
.prepare(
'SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL'
)
.get(decisionId, owner)
)
return;
throw new Error(ownershipMessage(owner));
}
/** Operator cancellation is a policy pause, not a fresh scaling signal. */
export function pauseScalingForOperation(operationId: string): void {
const row = db().prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?').get(operationId) as { policy_id: string } | undefined;
const row = db()
.prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?')
.get(operationId) as { policy_id: string } | undefined;
if (!row) return;
db().prepare("UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?").run(new Date().toISOString(), row.policy_id);
db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL').run(Date.now(), row.policy_id);
db()
.prepare(
"UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?"
)
.run(new Date().toISOString(), row.policy_id);
db()
.prepare(
'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL'
)
.run(Date.now(), row.policy_id);
}
+143 -45
View File
@@ -1,72 +1,170 @@
import type { ScalingDecision, ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import type {
ScalingDecision,
ScalingMember,
ScalingPolicy,
ScalingSnapshot
} from '../../autoscaling.js';
import { parseServerName, zombieMaps } from '../../map-aliases.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
export interface EmptyObservation { identity: string; observedAt: number | null; emptySince: number | null; evaluatedAt: number }
export interface EmptyObservation {
identity: string;
observedAt: number | null;
emptySince: number | null;
evaluatedAt: number;
}
/** Only distinct, continuous evidence can advance an empty interval. */
export function emptyObservation(member: ScalingMember, previous: EmptyObservation | undefined, now: number): EmptyObservation {
export function emptyObservation(
member: ScalingMember,
previous: EmptyObservation | undefined,
now: number
): EmptyObservation {
const stamp = member.observedAt;
const empty = member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
const continuous = empty && previous?.identity === member.identity && previous.emptySince !== null
&& (!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt)
&& previous.observedAt !== null && stamp >= previous.observedAt && stamp - previous.observedAt <= ONLINE_MAX_MS
&& now >= previous.evaluatedAt && now - previous.evaluatedAt <= ONLINE_MAX_MS;
return { identity: member.identity, observedAt: stamp, emptySince: empty ? continuous ? previous!.emptySince : stamp : null, evaluatedAt: now };
const empty =
member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
const continuous =
empty &&
previous?.identity === member.identity &&
previous.emptySince !== null &&
(!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt) &&
previous.observedAt !== null &&
stamp >= previous.observedAt &&
stamp - previous.observedAt <= ONLINE_MAX_MS &&
now >= previous.evaluatedAt &&
now - previous.evaluatedAt <= ONLINE_MAX_MS;
return {
identity: member.identity,
observedAt: stamp,
emptySince: empty ? (continuous ? previous!.emptySince : stamp) : null,
evaluatedAt: now
};
}
export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, now: number): ScalingDecision {
export function planScaling(
policy: ScalingPolicy,
snapshot: ScalingSnapshot,
now: number
): ScalingDecision {
if (policy.mapSlug === '*') {
const maps = zombieMaps(policy.game);
const decisions = maps.map(map => {
const matches = (name: string) => (snapshot.members.find(m => m.name === name)?.mapSlug ?? parseServerName(name)?.mapSlug) === map.slug;
return planScaling({ ...policy, mapSlug: map.slug }, {
...snapshot, members: snapshot.members.filter(m => matches(m.name)),
reservedNames: snapshot.reservedNames.filter(matches)
}, now);
const decisions = maps.map((map) => {
const matches = (name: string) =>
(snapshot.members.find((m) => m.name === name)?.mapSlug ??
parseServerName(name)?.mapSlug) === map.slug;
return planScaling(
{ ...policy, mapSlug: map.slug },
{
...snapshot,
members: snapshot.members.filter((m) => matches(m.name)),
reservedNames: snapshot.reservedNames.filter(matches)
},
now
);
});
// Serialize game-wide work and prefer replenishing any map before trimming another.
const pending = snapshot.members.find(m => m.state === 'creating' || m.state === 'removing');
const selected = decisions.find(d => d.action === 'create') ?? decisions.find(d => d.action === 'destroy')
?? decisions.find(d => d.deficit > 0 || d.unknown > 0) ?? decisions[0];
const pending = snapshot.members.find(
(m) => m.state === 'creating' || m.state === 'removing'
);
const selected =
decisions.find((d) => d.action === 'create') ??
decisions.find((d) => d.action === 'destroy') ??
decisions.find((d) => d.deficit > 0 || d.unknown > 0) ??
decisions[0];
const map = maps[decisions.indexOf(selected)];
return { ...selected, action: pending ? 'wait' : selected.action, server: pending ? null : selected.server,
reason: pending ? pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.` : `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0), deficit: decisions.reduce((n, d) => n + d.deficit, 0), occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0), pending: decisions.reduce((n, d) => n + d.pending, 0), unknown: decisions.reduce((n, d) => n + d.unknown, 0) };
return {
...selected,
action: pending ? 'wait' : selected.action,
server: pending ? null : selected.server,
reason: pending
? (pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.`)
: `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0),
deficit: decisions.reduce((n, d) => n + d.deficit, 0),
occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0),
pending: decisions.reduce((n, d) => n + d.pending, 0),
unknown: decisions.reduce((n, d) => n + d.unknown, 0)
};
}
const { members } = snapshot;
const occupied = members.filter(m => m.state === 'occupied').length, empty = members.filter(m => m.state === 'empty').length;
const pending = members.filter(m => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter(m => m.state === 'unknown' || m.state === 'stopped').length;
const occupied = members.filter((m) => m.state === 'occupied').length,
empty = members.filter((m) => m.state === 'empty').length;
const pending = members.filter((m) => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter((m) => m.state === 'unknown' || m.state === 'stopped').length;
const desired = Math.min(policy.maximum, Math.max(policy.minimum, occupied + policy.spares));
const result = (action: ScalingDecision['action'], reason: string, server: string | null = null): ScalingDecision =>
({ action, reason, server, desired, deficit: Math.max(0, policy.spares - empty), occupied, empty, pending, unknown });
const result = (
action: ScalingDecision['action'],
reason: string,
server: string | null = null
): ScalingDecision => ({
action,
reason,
server,
desired,
deficit: Math.max(0, policy.spares - empty),
occupied,
empty,
pending,
unknown
});
if (policy.mode === 'off') return result('wait', policy.pausedReason ?? 'Policy is off.');
if (policy.pausedReason) return result('wait', policy.pausedReason);
if (snapshot.blockers.length) return result('wait', snapshot.blockers[0]);
const blocked = members.find(m => !['occupied', 'empty'].includes(m.state));
if (blocked) return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
if (members.some(m => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS)) return result('wait', 'Waiting for fresh occupancy evidence.');
const blocked = members.find((m) => !['occupied', 'empty'].includes(m.state));
if (blocked)
return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
if (
members.some(
(m) => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS
)
)
return result('wait', 'Waiting for fresh occupancy evidence.');
if (occupied + empty < desired) {
if (snapshot.reservedNames.length >= policy.maximum) return result('wait', 'Maximum lobby reservations reached.');
if (!snapshot.keyPool.available) return result('wait', snapshot.keyPool.total
? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.`
: `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`);
const available = Array.from({ length: 20 }, (_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`)
.find(name => !snapshot.reservedNames.includes(name));
return available ? result('create', 'Replenish empty lobby capacity.', available) : result('wait', 'No unused lobby identity is available.');
if (snapshot.reservedNames.length >= policy.maximum)
return result('wait', 'Maximum lobby reservations reached.');
if (!snapshot.keyPool.available)
return result(
'wait',
snapshot.keyPool.total
? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.`
: `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`
);
const available = Array.from(
{ length: 20 },
(_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`
).find((name) => !snapshot.reservedNames.includes(name));
return available
? result('create', 'Replenish empty lobby capacity.', available)
: result('wait', 'No unused lobby identity is available.');
}
if (occupied + empty > desired && empty > policy.spares && occupied + empty > policy.minimum) {
// Protect the lowest-numbered spares before checking removal timers.
// Lobby 1 must survive even when a higher lobby has only just emptied.
const orderedEmpty = members.filter(m => m.state === 'empty').sort((a, b) =>
(parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) - (parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER)
|| a.name.localeCompare(b.name, 'en', { numeric: true }));
const orderedEmpty = members
.filter((m) => m.state === 'empty')
.sort(
(a, b) =>
(parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) -
(parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER) ||
a.name.localeCompare(b.name, 'en', { numeric: true })
);
const requiredEmpty = Math.max(policy.spares, policy.minimum - occupied);
const candidates = orderedEmpty.slice(requiredEmpty).reverse()
.filter(m => m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000);
if (candidates.length) return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
const candidates = orderedEmpty
.slice(requiredEmpty)
.reverse()
.filter(
(m) =>
m.emptySince !== null &&
m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000
);
if (candidates.length)
return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
return result('wait', 'Waiting for a continuous empty interval.');
}
return result('wait', empty < policy.spares ? 'Maximum reached; the empty spare target cannot be met.' : 'Empty spare target is satisfied.');
return result(
'wait',
empty < policy.spares
? 'Maximum reached; the empty spare target cannot be met.'
: 'Empty spare target is satisfied.'
);
}
+13 -5
View File
@@ -36,20 +36,28 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
`);
handle.transaction(() => {
const columns = handle.pragma('table_info(autoscaling_policies)') as { name: string }[];
if (columns.some(column => column.name === 'next_action_at')) {
if (columns.some((column) => column.name === 'next_action_at')) {
handle.exec('ALTER TABLE autoscaling_policies DROP COLUMN next_action_at');
}
handle.exec(`UPDATE autoscaling_policies SET settings=json_remove(settings,'$.cooldownMinutes')
WHERE json_type(settings,'$.cooldownMinutes') IS NOT NULL`);
// Removed modes must not silently gain permission to destroy lobbies.
// Keep ownership while delivered work settles through normal reconciliation.
handle.prepare(`UPDATE autoscaling_authorizations SET revoked_at=?
handle
.prepare(
`UPDATE autoscaling_authorizations SET revoked_at=?
WHERE claimed_at IS NULL AND revoked_at IS NULL AND policy_id IN
(SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`).run(Date.now());
handle.prepare(`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'),
(SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`
)
.run(Date.now());
handle
.prepare(
`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'),
revision=revision+1, paused_reason='This autoscaling mode was removed. Review and turn autoscaling On to resume.',
owns_map=CASE WHEN EXISTS (SELECT 1 FROM autoscaling_decisions WHERE policy_id=autoscaling_policies.id AND resolved=0)
THEN owns_map ELSE 0 END, updated_at=?
WHERE mode IN ('dry-run','create-only')`).run(new Date().toISOString());
WHERE mode IN ('dry-run','create-only')`
)
.run(new Date().toISOString());
})();
}
+189 -45
View File
@@ -7,74 +7,218 @@ import { freshStatusTimestamp } from '../../server-freshness.js';
import type { ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import { emptyObservation, type EmptyObservation } from './planner.js';
export function mapIdentity(server: { name: string; game?: string | null; map_slug?: string | null; map?: string | null }): { game: string; mapSlug: string } | null {
const parsed = parseServerName(server.name), game = server.game ?? parsed?.game;
export function mapIdentity(server: {
name: string;
game?: string | null;
map_slug?: string | null;
map?: string | null;
}): { game: string; mapSlug: string } | null {
const parsed = parseServerName(server.name),
game = server.game ?? parsed?.game;
if (!game) return null;
if (server.map_slug && zombieMaps(game).some(m => m.slug === server.map_slug)) return { game, mapSlug: server.map_slug };
if (server.map_slug && zombieMaps(game).some((m) => m.slug === server.map_slug))
return { game, mapSlug: server.map_slug };
if (server.map_slug) return null;
if (parsed?.game === game && parsed.mapCanonical && (!server.map || parsed.mapCanonical === server.map)) return { game, mapSlug: parsed.mapSlug };
const matches = zombieMaps(game).filter(m => m.canonical === server.map);
if (
parsed?.game === game &&
parsed.mapCanonical &&
(!server.map || parsed.mapCanonical === server.map)
)
return { game, mapSlug: parsed.mapSlug };
const matches = zombieMaps(game).filter((m) => m.canonical === server.map);
return matches.length === 1 ? { game, mapSlug: matches[0].slug } : null;
}
export function protectedWork(server: string, exceptOperation = ''): string | null {
const handle = db();
if (handle.prepare(`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).get(server)) return 'Active beta';
if (handle.prepare(`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).get(server)) return 'Campaign work';
if (handle.prepare(`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`).get(server)) return 'Maintenance work';
if (handle.prepare(`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`).get(server, exceptOperation)) return 'Unresolved recovery';
if (
handle
.prepare(
`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`
)
.get(server)
)
return 'Active beta';
if (
handle
.prepare(
`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`
)
.get(server)
)
return 'Campaign work';
if (
handle
.prepare(
`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`
)
.get(server)
)
return 'Maintenance work';
if (
handle
.prepare(
`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`
)
.get(server, exceptOperation)
)
return 'Unresolved recovery';
return null;
}
export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptOperation = ''): ScalingSnapshot {
const handle = db(), observe = observationReader();
export function scalingSnapshot(
policy: ScalingPolicy,
now = Date.now(),
exceptOperation = ''
): ScalingSnapshot {
const handle = db(),
observe = observationReader();
const rows = listServers();
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && (policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug); };
const matches = (value: Parameters<typeof mapIdentity>[0]) => {
const identity = mapIdentity(value);
return (
identity?.game === policy.game &&
(policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug)
);
};
const blockers: string[] = [];
for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string; stage: string }[];
const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
for (const row of rows)
if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game)
blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle
.prepare(
`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`
)
.all(exceptOperation) as {
id: string;
server: string;
action: string;
state: string;
stage: string;
}[];
const reservations = handle
.prepare(
`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`
)
.all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => {
const observation = observe(row), operation = operations.find(o => o.server === row.name);
const observation = observe(row),
operation = operations.find((o) => o.server === row.name);
const protection = protectedWork(row.name, exceptOperation);
const host = row.host_id ? handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as { mode: string } | undefined : undefined;
const host = row.host_id
? (handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as
{ mode: string } | undefined)
: undefined;
const identity = `${row.native_instance_id ?? row.created_at}:${row.instance_generation ?? 0}:${observation.telemetry_after ?? ''}`;
let state: ScalingMember['state'] = 'unknown', blocker: string | null = null;
let state: ScalingMember['state'] = 'unknown',
blocker: string | null = null;
let players = observation.health === 'online' ? observation.last_players : null;
let observedAt = players !== null && observation.last_seen_at ? Date.parse(observation.last_seen_at) : null;
let observedAt =
players !== null && observation.last_seen_at
? Date.parse(observation.last_seen_at)
: null;
let source = observation.status_source;
// Display evidence prefers telemetry; scaling additionally honors a current positive RCON veto.
if ((row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now)
&& (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) {
players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon';
if (
(row.last_players ?? 0) > 0 &&
freshStatusTimestamp(row.last_seen_at, now) &&
(!observation.telemetry_after ||
Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))
) {
players = row.last_players;
observedAt = Date.parse(row.last_seen_at!);
source = 'rcon';
}
if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; }
else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id}: ${operation.stage}`; }
else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
return { name: row.name, mapSlug: mapIdentity(row)!.mapSlug, identity, state, players, observedAt, source: source ?? null, emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
if (protection || !row.native_instance_id || host?.mode !== 'active') {
state = 'protected';
blocker = protection ?? 'Native instance on an active host required';
} else if (operation) {
state =
operation.action === 'create'
? 'creating'
: operation.action === 'destroy'
? 'removing'
: 'protected';
blocker = `Provisioning ${operation.id}: ${operation.stage}`;
} else if (
row.desired_state === 'stopped' ||
['stopped', 'exited', 'created', 'dead'].includes(row.container_state ?? '')
)
state = 'stopped';
else if (
observation.health === 'online' &&
players !== null &&
observedAt !== null &&
observedAt <= now &&
freshStatusTimestamp(new Date(observedAt).toISOString(), now)
)
state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since]
.map((at) => Date.parse(at ?? ''))
.filter((at) => Number.isFinite(at) && at <= now);
return {
name: row.name,
mapSlug: mapIdentity(row)!.mapSlug,
identity,
state,
players,
observedAt,
source: source ?? null,
emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null,
blocker,
operationId: operation?.id ?? null
};
});
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
const operation = operations.find(o => o.server === row.name);
members.push({ name: row.name, mapSlug: row.map_slug, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
}
const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
for (const row of reservations.filter(matches))
if (!members.some((m) => m.name === row.name)) {
const operation = operations.find((o) => o.server === row.name);
members.push({
name: row.name,
mapSlug: row.map_slug,
identity: 'reserved',
state: 'creating',
players: null,
observedAt: null,
source: null,
emptySince: null,
blocker: operation?.stage ?? 'Creation is pending.',
operationId: operation?.id ?? null
});
}
const retained = handle
.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?')
.all(policy.id) as { server: string; observation: string }[];
for (const member of members) {
const saved = retained.find(r => r.server === member.name);
const previous = saved ? JSON.parse(saved.observation) as EmptyObservation : undefined;
const saved = retained.find((r) => r.server === member.name);
const previous = saved ? (JSON.parse(saved.observation) as EmptyObservation) : undefined;
member.emptySince = emptyObservation(member, previous, now).emptySince;
}
const reservedNames = new Set([...rows.filter(matches).map(r => r.name), ...reservations.filter(matches).map(r => r.name)]);
for (const row of handle.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL').all() as { name: string }[]) if (matches(row)) {
reservedNames.add(row.name);
if (!members.some(m => m.name === row.name)) blockers.push(`Unresolved native instance ${row.name}.`);
}
for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as { name: string }[]) if (matches(row)) reservedNames.add(row.name);
const ownServer = exceptOperation ? (handle.prepare('SELECT server FROM server_deployments WHERE id=?').get(exceptOperation) as { server: string } | undefined)?.server : undefined;
const reservedNames = new Set([
...rows.filter(matches).map((r) => r.name),
...reservations.filter(matches).map((r) => r.name)
]);
for (const row of handle
.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL')
.all() as { name: string }[])
if (matches(row)) {
reservedNames.add(row.name);
if (!members.some((m) => m.name === row.name))
blockers.push(`Unresolved native instance ${row.name}.`);
}
for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as {
name: string;
}[])
if (matches(row)) reservedNames.add(row.name);
const ownServer = exceptOperation
? (
handle
.prepare('SELECT server FROM server_deployments WHERE id=?')
.get(exceptOperation) as { server: string } | undefined
)?.server
: undefined;
const { total, available } = serverKeyPool(policy.game, ownServer);
return { members, reservedNames: [...reservedNames], keyPool: { total, available }, blockers };
}
+107 -29
View File
@@ -16,28 +16,65 @@ function remote(path: string) {
}
return null;
}
function missing(path: string): never { throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' }); }
function missing(path: string): never {
throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' });
}
export interface FileFingerprint { size: number; sha256: string }
export interface FileFingerprint {
size: number;
sha256: string;
}
/** One scoped agent scan; null preserves compatibility with older agents and local files. */
export async function nativeFileManifest(root: string, directories: string[], ignoreNames: string[], managedPaths: string[]): Promise<Map<string, FileFingerprint> | null> {
export async function nativeFileManifest(
root: string,
directories: string[],
ignoreNames: string[],
managedPaths: string[]
): Promise<Map<string, FileFingerprint> | null> {
const target = remote(root);
if (!target || target.path !== '.' || !getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest')) return null;
const roots = [...new Set(directories.map(directory => relative(root, directory).split(sep).join('/')))];
if (
!target ||
target.path !== '.' ||
!getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest')
)
return null;
const roots = [
...new Set(directories.map((directory) => relative(root, directory).split(sep).join('/')))
];
// Preserve legacy configurations that scan the entire instance root.
if (roots.some(directory => !directory)) return null;
const paths = [...new Set(managedPaths.map(path => relative(root, path).split(sep).join('/')))];
const result = await nativeRequest<{ files: Array<FileFingerprint & { path: string }> }>(target.instance, 'files.manifest', {
roots, paths, ignoreNames: ignoreNames.filter(name => !paths.some(path => path.split('/').includes(name)))
});
if (!result || !Array.isArray(result.files) || result.files.length > 10_000) throw new Error('Agent returned an invalid file manifest.');
if (roots.some((directory) => !directory)) return null;
const paths = [
...new Set(managedPaths.map((path) => relative(root, path).split(sep).join('/')))
];
const result = await nativeRequest<{ files: Array<FileFingerprint & { path: string }> }>(
target.instance,
'files.manifest',
{
roots,
paths,
ignoreNames: ignoreNames.filter(
(name) => !paths.some((path) => path.split('/').includes(name))
)
}
);
if (!result || !Array.isArray(result.files) || result.files.length > 10_000)
throw new Error('Agent returned an invalid file manifest.');
const manifest = new Map<string, FileFingerprint>();
for (const entry of result.files) {
if (!entry || typeof entry.path !== 'string' || !entry.path || entry.path.includes('\\') || entry.path.includes('\0')
|| posix.normalize(entry.path) !== entry.path || entry.path.split('/').some(part => part === '..' || part === '.')
|| !roots.some(directory => entry.path.startsWith(`${directory}/`))
|| !Number.isSafeInteger(entry.size) || entry.size < 0 || !/^[a-f0-9]{64}$/.test(entry.sha256)) {
if (
!entry ||
typeof entry.path !== 'string' ||
!entry.path ||
entry.path.includes('\\') ||
entry.path.includes('\0') ||
posix.normalize(entry.path) !== entry.path ||
entry.path.split('/').some((part) => part === '..' || part === '.') ||
!roots.some((directory) => entry.path.startsWith(`${directory}/`)) ||
!Number.isSafeInteger(entry.size) ||
entry.size < 0 ||
!/^[a-f0-9]{64}$/.test(entry.sha256)
) {
throw new Error('Agent returned an invalid file manifest entry.');
}
const path = join(root, entry.path);
@@ -50,44 +87,85 @@ export async function nativeFileManifest(root: string, directories: string[], ig
export async function readFile(path: string): Promise<Buffer> {
const target = remote(path);
if (!target) return fs.readFile(path);
const result = await nativeRequest<{ contentSha256?: string; contentBase64?: string | null } | null>(target.instance, 'files.read', { path: target.path });
const result = await nativeRequest<{
contentSha256?: string;
contentBase64?: string | null;
} | null>(target.instance, 'files.read', { path: target.path });
// Stored receipts reference shared bytes; read them directly without a base64 round trip.
if (typeof result?.contentSha256 === 'string') return getFileBlob(result.contentSha256);
if (result?.contentBase64 === null) return missing(path);
if (typeof result?.contentBase64 === 'string') return Buffer.from(result.contentBase64, 'base64');
if (typeof result?.contentBase64 === 'string')
return Buffer.from(result.contentBase64, 'base64');
throw new Error(`Agent returned an invalid response for files.read: ${target.path}`);
}
export async function exists(path: string): Promise<boolean> {
try { await stat(path); return true; }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; }
try {
await stat(path);
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
throw error;
}
}
export async function stat(path: string): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> {
export async function stat(
path: string
): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> {
const target = remote(path);
if (!target) return fs.stat(path);
const result = await nativeRequest<{ exists?: boolean; size: number; mtime: string | number; isDirectory: boolean; isFile: boolean } | null>(target.instance, 'files.stat', { path: target.path });
const result = await nativeRequest<{
exists?: boolean;
size: number;
mtime: string | number;
isDirectory: boolean;
isFile: boolean;
} | null>(target.instance, 'files.stat', { path: target.path });
if (result === null || result.exists === false) return missing(path);
return { size: result.size, mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime), isDirectory: () => result.isDirectory, isFile: () => result.isFile };
return {
size: result.size,
mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime),
isDirectory: () => result.isDirectory,
isFile: () => result.isFile
};
}
export async function readdir(path: string, options: { withFileTypes: true }): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> {
export async function readdir(
path: string,
options: { withFileTypes: true }
): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> {
const target = remote(path);
if (!target) return fs.readdir(path, options);
const result = await nativeRequest<{ entries: { name: string; isDirectory: boolean; isFile?: boolean }[] }>(target.instance, 'files.list', { path: target.path });
return result.entries.map(entry => ({ name: entry.name, isDirectory: () => entry.isDirectory, isFile: () => entry.isFile ?? !entry.isDirectory }));
const result = await nativeRequest<{
entries: { name: string; isDirectory: boolean; isFile?: boolean }[];
}>(target.instance, 'files.list', { path: target.path });
return result.entries.map((entry) => ({
name: entry.name,
isDirectory: () => entry.isDirectory,
isFile: () => entry.isFile ?? !entry.isDirectory
}));
}
export async function nativeTail(path: string, lines: number) {
const target = remote(path);
if (!target) return undefined;
return nativeRequest<import('../fleet/logs.js').TailResult | null>(target.instance, 'logs.read', { path: target.path, lines });
return nativeRequest<import('../fleet/logs.js').TailResult | null>(
target.instance,
'logs.read',
{ path: target.path, lines }
);
}
export async function nativeLogChunk(path: string, cursor?: import('../fleet/logs.js').LogCursor) {
const target = remote(path);
if (!target) return undefined;
const incremental = getHost(target.instance.host_id)?.inventory.capabilities?.includes('logs.read-offset');
return nativeRequest<import('../fleet/logs.js').LogChunk | import('../fleet/logs.js').TailResult | null>(target.instance, 'logs.read',
{ path: target.path, ...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 }) });
const incremental = getHost(target.instance.host_id)?.inventory.capabilities?.includes(
'logs.read-offset'
);
return nativeRequest<
import('../fleet/logs.js').LogChunk | import('../fleet/logs.js').TailResult | null
>(target.instance, 'logs.read', {
path: target.path,
...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 })
});
}
+105 -28
View File
@@ -6,13 +6,26 @@ import { trackActivity } from '../pipeline/activity.js';
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
export interface NativeInstance {
id: string; server: string; host_id: string; generation: number; library_id: string;
image: string; bind_address: string; port: number; public_address: string | null;
public_port: number | null; created_at: string; retired_at: string | null;
id: string;
server: string;
host_id: string;
generation: number;
library_id: string;
image: string;
bind_address: string;
port: number;
public_address: string | null;
public_port: number | null;
created_at: string;
retired_at: string | null;
}
export function nativeInstance(server: string): NativeInstance | null {
return db().prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL').get(server) as NativeInstance | undefined ?? null;
return (
(db()
.prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL')
.get(server) as NativeInstance | undefined) ?? null
);
}
/** Credentials come from the launch, never from the possibly modified config or
@@ -21,72 +34,136 @@ export function nativeInstance(server: string): NativeInstance | null {
export function runtimePasswordForServer(server: string): string | null {
const instance = nativeInstance(server);
if (!instance) return null;
const row = db().prepare(`SELECT
const row = db()
.prepare(
`SELECT
json_extract(COALESCE(p.payload,o.payload),'$.rconPassword') AS password,
json_extract(COALESCE(p.payload,o.payload),'$.env.PLUTAINER_RCON_PASSWORD') AS env_password
FROM ec_host_operations o JOIN native_instances i ON ${runtimeLaunchSql}
LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id
WHERE i.id=? AND o.state='completed' AND o.history_expired_at IS NULL
ORDER BY o.rowid DESC LIMIT 1`).get(instance.id) as { password: unknown; env_password: unknown } | undefined;
if (!row) throw new Error(`Cannot verify runtime config for ${server}: the current instance's launch credentials are unavailable.`);
ORDER BY o.rowid DESC LIMIT 1`
)
.get(instance.id) as { password: unknown; env_password: unknown } | undefined;
if (!row)
throw new Error(
`Cannot verify runtime config for ${server}: the current instance's launch credentials are unavailable.`
);
const password = row.password || row.env_password;
return typeof password === 'string' && password ? password : null;
}
export function nativeDeployment(id: string): boolean {
return !!db().prepare(`SELECT 1 FROM native_provisioning WHERE deployment_id = ?
return !!db()
.prepare(
`SELECT 1 FROM native_provisioning WHERE deployment_id = ?
UNION ALL SELECT 1 FROM native_creation_intents WHERE deployment_id = ?
UNION ALL SELECT 1 FROM native_creation_cancellations c
JOIN native_creation_intents i ON i.deployment_id=c.creation_id WHERE c.cleanup_id = ? LIMIT 1`).get(id, id, id);
JOIN native_creation_intents i ON i.deployment_id=c.creation_id WHERE c.cleanup_id = ? LIMIT 1`
)
.get(id, id, id);
}
export function unplacedNativeCreation(server: string): string | null {
const row = db().prepare(`SELECT d.id FROM server_deployments d
const row = db()
.prepare(
`SELECT d.id FROM server_deployments d
JOIN native_creation_intents i ON i.deployment_id=d.id
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
WHERE d.server=? AND d.server_deleted_at IS NULL AND d.action='create' AND d.state='queued'
AND n.deployment_id IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(server) as { id: string } | undefined;
AND n.deployment_id IS NULL ORDER BY d.rowid DESC LIMIT 1`
)
.get(server) as { id: string } | undefined;
return row?.id ?? null;
}
const observations = new Map<string, Promise<unknown>>();
const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'files.manifest', 'logs.read']);
const readKinds = new Set([
'instance.inspect',
'files.read',
'files.stat',
'files.list',
'files.manifest',
'logs.read'
]);
/** Share overlapping observations only, never completed results or mutations. */
export async function nativeRequest<T = Record<string, unknown>>(instance: NativeInstance, kind: string, payload: Record<string, unknown> = {}, key?: string): Promise<T> {
export async function nativeRequest<T = Record<string, unknown>>(
instance: NativeInstance,
kind: string,
payload: Record<string, unknown> = {},
key?: string
): Promise<T> {
const host = getHost(instance.host_id);
if (!host || host.connectionState !== 'online') throw new Error('Host agent is offline; instance state is unknown.');
const readOnly = readKinds.has(kind) || (kind === 'rcon.execute' && ['status', 'getstatus', 'version'].includes(String(payload.command)));
if (!host || host.connectionState !== 'online')
throw new Error('Host agent is offline; instance state is unknown.');
const readOnly =
readKinds.has(kind) ||
(kind === 'rcon.execute' &&
['status', 'getstatus', 'version'].includes(String(payload.command)));
if (key || !readOnly) return executeRequest<T>(instance, kind, payload, key);
// A probe after a lifecycle submission must not join an old-process read.
const identity = JSON.stringify([instance.host_id, instance.id, instance.generation, instance.server,
latestGameSubmission(instance.server), kind, payload]);
const identity = JSON.stringify([
instance.host_id,
instance.id,
instance.generation,
instance.server,
latestGameSubmission(instance.server),
kind,
payload
]);
const pending = observations.get(identity);
if (pending) return pending as Promise<T>;
const request = executeRequest<T>(instance, kind, payload).finally(() => { observations.delete(identity); });
const request = executeRequest<T>(instance, kind, payload).finally(() => {
observations.delete(identity);
});
observations.set(identity, request);
return request;
}
/** Only the agent receives host paths, Docker access, or game credentials. */
async function executeRequest<T>(instance: NativeInstance, kind: string, payload: Record<string, unknown>, key?: string): Promise<T> {
async function executeRequest<T>(
instance: NativeInstance,
kind: string,
payload: Record<string, unknown>,
key?: string
): Promise<T> {
const operation = queueHostOperation(instance.host_id, kind, payload, {
instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID()
instanceId: instance.id,
generation: instance.generation,
idempotencyKey: key ?? randomUUID()
});
const result = await trackActivity(`agent:${operation.id}`, `Waiting for ${kind} on ${instance.server}`,
() => waitHostOperationStatus(operation.id, 25_000));
if (result.state !== 'completed') throw new Error(result.error || `Agent operation ${result.id} is ${result.state}; its original ID is retained.`);
const result = await trackActivity(
`agent:${operation.id}`,
`Waiting for ${kind} on ${instance.server}`,
() => waitHostOperationStatus(operation.id, 25_000)
);
if (result.state !== 'completed')
throw new Error(
result.error ||
`Agent operation ${result.id} is ${result.state}; its original ID is retained.`
);
// Do not apply replies from a retired generation to a replacement server.
const current = nativeInstance(instance.server);
if (current?.id !== instance.id || current.generation !== instance.generation) throw new Error('Instance was replaced during the operation.');
if (current?.id !== instance.id || current.generation !== instance.generation)
throw new Error('Instance was replaced during the operation.');
return result.result as T;
}
export async function nativeHealth(server: string): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> {
export async function nativeHealth(
server: string
): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> {
const instance = nativeInstance(server);
if (!instance) return { ok: false, reason: 'Native instance is not registered' };
try {
const result = await nativeRequest<{ state: string; healthy: boolean }>(instance, 'instance.inspect');
return result.healthy ? { ok: true, state: result.state } : { ok: false, state: result.state, reason: `Container ${result.state}` };
} catch (error) { return { ok: false, state: 'unknown', reason: (error as Error).message }; }
const result = await nativeRequest<{ state: string; healthy: boolean }>(
instance,
'instance.inspect'
);
return result.healthy
? { ok: true, state: result.state }
: { ok: false, state: result.state, reason: `Container ${result.state}` };
} catch (error) {
return { ok: false, state: 'unknown', reason: (error as Error).message };
}
}
+36 -12
View File
@@ -43,11 +43,21 @@ export function migrateNativeInstances(handle: Database.Database): void {
);
CREATE INDEX IF NOT EXISTS idx_ec_host_migrations_instance ON ec_host_migrations(instance_id);
`);
if (!(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some(column => column.name === 'runtime_password')) {
if (
!(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some(
(column) => column.name === 'runtime_password'
)
) {
handle.exec('ALTER TABLE ec_host_migrations ADD COLUMN runtime_password TEXT');
}
if (!(handle.pragma('table_info(native_provisioning)') as { name: string }[]).some(column => column.name === 'recovery_step')) {
handle.exec("ALTER TABLE native_provisioning ADD COLUMN recovery_step TEXT CHECK (recovery_step IN ('restart','rebuild'))");
if (
!(handle.pragma('table_info(native_provisioning)') as { name: string }[]).some(
(column) => column.name === 'recovery_step'
)
) {
handle.exec(
"ALTER TABLE native_provisioning ADD COLUMN recovery_step TEXT CHECK (recovery_step IN ('restart','rebuild'))"
);
}
if (!handle.prepare("SELECT 1 FROM settings WHERE key='native_file_plans_v1'").get()) {
handle.transaction(() => {
@@ -56,29 +66,43 @@ export function migrateNativeInstances(handle: Database.Database): void {
INSERT INTO settings(key,value) VALUES('native_file_plans_v1','1');`);
})();
}
}
/** Accepted native creations receive their host port only when provisioning places them. */
function migrateUnplacedCreationPort(handle: Database.Database): void {
const columns = handle.prepare('PRAGMA table_info(server_creation_jobs)').all() as { name: string; notnull: number }[];
if (!columns.find(column => column.name === 'port')?.notnull) return;
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='server_creation_jobs'").get() as { sql: string };
const columns = handle.prepare('PRAGMA table_info(server_creation_jobs)').all() as {
name: string;
notnull: number;
}[];
if (!columns.find((column) => column.name === 'port')?.notnull) return;
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='server_creation_jobs'")
.get() as { sql: string };
const replacement = sql.replace(/\bport\s+INTEGER\s+NOT\s+NULL\b/i, 'port INTEGER');
if (replacement === sql) throw new Error('Could not migrate unplaced creation ports.');
const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='server_creation_jobs' AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependent = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name='server_creation_jobs' AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all() as { sql: string }[];
const foreignKeys = handle.pragma('foreign_keys', { simple: true });
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement.replace('server_creation_jobs', 'server_creation_jobs_unplaced'));
const names = columns.map(column => `"${column.name.replaceAll('"', '""')}"`).join(', ');
handle.exec(
replacement.replace('server_creation_jobs', 'server_creation_jobs_unplaced')
);
const names = columns
.map((column) => `"${column.name.replaceAll('"', '""')}"`)
.join(', ');
handle.exec(`INSERT INTO server_creation_jobs_unplaced(rowid,${names}) SELECT rowid,${names} FROM server_creation_jobs;
DROP TABLE server_creation_jobs;
ALTER TABLE server_creation_jobs_unplaced RENAME TO server_creation_jobs;`);
for (const object of dependent) handle.exec(object.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Unplaced creation migration found broken references.');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Unplaced creation migration found broken references.');
})();
} finally { handle.pragma(`foreign_keys = ${foreignKeys ? 'ON' : 'OFF'}`); }
} finally {
handle.pragma(`foreign_keys = ${foreignKeys ? 'ON' : 'OFF'}`);
}
}
+17 -9
View File
@@ -17,13 +17,15 @@ function num(name: string, fallback: number): number {
export const config = {
serversRoot: req('SERVERS_ROOT', '/root/zombies/t5'),
t6ServersRoot: env.T6_SERVERS_ROOT ?? join(dirname(req('SERVERS_ROOT', '/root/zombies/t5')), 't6'),
t6ServersRoot:
env.T6_SERVERS_ROOT ?? join(dirname(req('SERVERS_ROOT', '/root/zombies/t5')), 't6'),
t6RelDestScript: env.T6_REL_DEST_SCRIPT ?? 'runtime/plutonium/storage/t6/scripts/zm',
t6ConfigTargetFile: env.T6_CONFIG_TARGET_FILE ?? 'dedicated_zm.cfg',
relDest: req('REL_DEST', 'runtime/plutonium/storage/t5/raw/scripts/sp'),
// Per-kind destination sub-paths under each server's directory. `relDest`
// above stays as the script fallback so existing deploys keep working.
relDestScript: env.REL_DEST_SCRIPT ?? env.REL_DEST ?? 'runtime/plutonium/storage/t5/raw/scripts/sp',
relDestScript:
env.REL_DEST_SCRIPT ?? env.REL_DEST ?? 'runtime/plutonium/storage/t5/raw/scripts/sp',
relDestConfig: env.REL_DEST_CONFIG ?? 'configs',
relDestPlugin: env.REL_DEST_PLUGIN ?? 'runtime/plutonium/plugins',
// Legacy T5 config filename. T6 has its own target above.
@@ -54,19 +56,25 @@ export type ArtifactKind = 'script' | 'config' | 'plugin' | 'baseline';
export function relDestForKind(kind: ArtifactKind, game = 't5'): string {
switch (kind) {
case 'script': return game === 't6' ? config.t6RelDestScript : config.relDestScript;
case 'config': return config.relDestConfig;
case 'baseline': return config.relDestConfig;
case 'plugin': return config.relDestPlugin;
case 'script':
return game === 't6' ? config.t6RelDestScript : config.relDestScript;
case 'config':
return config.relDestConfig;
case 'baseline':
return config.relDestConfig;
case 'plugin':
return config.relDestPlugin;
}
}
// Server IDs keep the game prefix even while creation is queued and no registry
// row exists yet. Legacy IDs retain the original T5 filesystem defaults.
export const gameOfServer = (server: string) => server.startsWith('t6_') ? 't6' : 't5';
export const rootOfServer = (server: string) => gameOfServer(server) === 't6' ? config.t6ServersRoot : config.serversRoot;
export const gameOfServer = (server: string) => (server.startsWith('t6_') ? 't6' : 't5');
export const rootOfServer = (server: string) =>
gameOfServer(server) === 't6' ? config.t6ServersRoot : config.serversRoot;
export const serverDir = (server: string) => `${rootOfServer(server)}/${server}`;
export const configFileForGame = (game: string | null) => game === 't6' ? config.t6ConfigTargetFile : config.configTargetFile;
export const configFileForGame = (game: string | null) =>
game === 't6' ? config.t6ConfigTargetFile : config.configTargetFile;
export const configFileOf = (server: string) => configFileForGame(gameOfServer(server));
export const destOfKind = (server: string, kind: ArtifactKind) =>
`${serverDir(server)}/${relDestForKind(kind, gameOfServer(server))}`;
+4 -2
View File
@@ -26,8 +26,10 @@ export function migrationStep<T>(name: string, work: () => T): T {
console.info(`[database] migration starting: ${name}`);
try {
const result = work();
console.info(`[database] migration finished: ${name}`, { elapsedMs: Date.now() - started,
rssMiB: Math.round(process.memoryUsage().rss / 1048576) });
console.info(`[database] migration finished: ${name}`, {
elapsedMs: Date.now() - started,
rssMiB: Math.round(process.memoryUsage().rss / 1048576)
});
return result;
} catch (error) {
console.error(`[database] migration failed: ${name}`, error);
+101 -39
View File
@@ -10,67 +10,103 @@ export function ensureFileBlobSchema(handle: Database.Database): void {
CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256);`);
}
export function putFileBlob(content: Uint8Array | null, handle: Database.Database = db()): string | null {
export function putFileBlob(
content: Uint8Array | null,
handle: Database.Database = db()
): string | null {
if (content === null) return null;
const sha256 = createHash('sha256').update(content).digest('hex');
handle.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)').run(sha256, content.byteLength, content);
handle
.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)')
.run(sha256, content.byteLength, content);
return sha256;
}
export function getFileBlob(sha256: string, handle: Database.Database = db()): Buffer {
const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as { content: Buffer } | undefined;
const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as
{ content: Buffer } | undefined;
if (!row) throw new Error('File content is unavailable or has expired.');
return row.content;
}
/** SQL fragments are for fixed schema identifiers, never user input. */
export const blobContent = (column: string) => `(SELECT content FROM file_blobs WHERE sha256=${column})`;
export const blobContent = (column: string) =>
`(SELECT content FROM file_blobs WHERE sha256=${column})`;
export const blobSize = (column: string) => `(SELECT size FROM file_blobs WHERE sha256=${column})`;
export function referenceBlobs(ownerType: string, ownerId: string, hashes: Iterable<string>, handle: Database.Database = db()): void {
handle.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?').run(ownerType, ownerId);
const insert = handle.prepare('INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)');
export function referenceBlobs(
ownerType: string,
ownerId: string,
hashes: Iterable<string>,
handle: Database.Database = db()
): void {
handle
.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?')
.run(ownerType, ownerId);
const insert = handle.prepare(
'INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)'
);
for (const hash of hashes) insert.run(ownerType, ownerId, hash);
}
/** Immutable file bytes in JSON are replaced with hash references, including before images. */
export function packFileJson(value: unknown, handle: Database.Database = db(), hashes = new Set<string>()): { value: unknown; hashes: Set<string> } {
if (Array.isArray(value)) return { value: value.map(item => packFileJson(item, handle, hashes).value), hashes };
export function packFileJson(
value: unknown,
handle: Database.Database = db(),
hashes = new Set<string>()
): { value: unknown; hashes: Set<string> } {
if (Array.isArray(value))
return { value: value.map((item) => packFileJson(item, handle, hashes).value), hashes };
if (!value || typeof value !== 'object') return { value, hashes };
const result: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) {
if ((key === 'contentBase64' || key === 'content' && 'path' in value && 'sha256' in value) && typeof item === 'string') {
if (
(key === 'contentBase64' ||
(key === 'content' && 'path' in value && 'sha256' in value)) &&
typeof item === 'string'
) {
const hash = putFileBlob(Buffer.from(item, 'base64'), handle)!;
result[key === 'content' ? 'contentHash' : 'contentSha256'] = hash;
hashes.add(hash);
} else {
result[key] = packFileJson(item, handle, hashes).value;
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') hashes.add(item);
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string')
hashes.add(item);
}
}
return { value: result, hashes };
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Unpacked JSON retains the caller-specific stored object shape.
export function unpackFileJson(value: unknown, handle: Database.Database = db()): any {
if (Array.isArray(value)) return value.map(item => unpackFileJson(item, handle));
if (Array.isArray(value)) return value.map((item) => unpackFileJson(item, handle));
if (!value || typeof value !== 'object') return value;
const result: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) {
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') {
result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(item, handle).toString('base64');
result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(
item,
handle
).toString('base64');
} else result[key] = unpackFileJson(item, handle);
}
return result;
}
export const blobColumns = [
['script_versions', 'content'], ['staged_file_changes', 'content'],
['deployment_files', 'content_before'], ['deployment_files', 'content_after'],
['beta_file_snapshots', 'prior_content'], ['beta_revision_files', 'content']
['script_versions', 'content'],
['staged_file_changes', 'content'],
['deployment_files', 'content_before'],
['deployment_files', 'content_after'],
['beta_file_snapshots', 'prior_content'],
['beta_revision_files', 'content']
] as const;
/** Finish each SELECT before writing; better-sqlite3 iterators hold an active cursor. */
export function* migrationRows<T>(handle: Database.Database, sql: string): Generator<T & { rid: number }> {
export function* migrationRows<T>(
handle: Database.Database,
sql: string
): Generator<T & { rid: number }> {
const select = handle.prepare(sql);
let cursor = -Number.MAX_SAFE_INTEGER;
for (;;) {
@@ -84,31 +120,53 @@ export function* migrationRows<T>(handle: Database.Database, sql: string): Gener
/** Backfill one row at a time so large libraries do not exhaust startup memory. */
export function migrateFileBlobs(handle: Database.Database): void {
for (const [table, column] of blobColumns) {
if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue;
migrationStep(`shared files: ${table}.${column}`, () => handle.transaction(() => {
handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`);
const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) {
update.run(putFileBlob(row.content, handle), row.rid);
}
handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
})());
if (
!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(
(item) => item.name === column
)
)
continue;
migrationStep(`shared files: ${table}.${column}`, () =>
handle.transaction(() => {
handle.exec(
`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`
);
const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
for (const row of migrationRows<{ content: Buffer | null }>(
handle,
`SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`
)) {
update.run(putFileBlob(row.content, handle), row.rid);
}
handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
})()
);
}
for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`);
for (const [table, column] of blobColumns)
handle.exec(
`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`
);
if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) {
migrationStep('shared files: launch recipes', () => handle.transaction(() => {
for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle,
'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) {
const packed = packFileJson(JSON.parse(row.recipe), handle);
handle.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(packed.value), row.deployment_id);
referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
}
handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
})());
migrationStep('shared files: launch recipes', () =>
handle.transaction(() => {
for (const row of migrationRows<{ deployment_id: string; recipe: string }>(
handle,
'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1'
)) {
const packed = packFileJson(JSON.parse(row.recipe), handle);
handle
.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?')
.run(JSON.stringify(packed.value), row.deployment_id);
referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
}
handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
})()
);
}
handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN
DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`);
for (const action of ['INSERT', 'UPDATE OF recipe']) handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'}
for (const action of ['INSERT', 'UPDATE OF recipe'])
handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'}
AFTER ${action} ON server_launch_recipes BEGIN
DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=NEW.deployment_id;
INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256)
@@ -117,8 +175,12 @@ export function migrateFileBlobs(handle: Database.Database): void {
}
export function pruneFileBlobs(limit = 100): number {
return db().prepare(`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b
return db()
.prepare(
`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b
WHERE NOT EXISTS (SELECT 1 FROM file_blob_references r WHERE r.sha256=b.sha256)
${blobColumns.map(([table, column]) => `AND NOT EXISTS (SELECT 1 FROM ${table} WHERE ${column}_hash=b.sha256)`).join('\n')}
LIMIT ?)`).run(limit).changes;
LIMIT ?)`
)
.run(limit).changes;
}
+15 -5
View File
@@ -1,26 +1,36 @@
import type Database from 'better-sqlite3';
import { defaultFleetSettings, validateFleetSettings, type FleetSettings } from '../../fleet-settings.js';
import {
defaultFleetSettings,
validateFleetSettings,
type FleetSettings
} from '../../fleet-settings.js';
export const FLEET_SETTINGS_KEY = 'fleet_settings';
/** Import legacy environment values once; persisted settings win on every later boot. */
export function initializeFleetSettings(handle: Database.Database): void {
const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
const saved = handle
.prepare('SELECT value FROM settings WHERE key = ?')
.get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
if (saved) {
const settings = JSON.parse(saved.value);
const upgraded = validateFleetSettings({ ...defaultFleetSettings, ...settings });
handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
handle
.prepare('UPDATE settings SET value = ? WHERE key = ?')
.run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
return;
}
const values = { ...defaultFleetSettings };
const legacy: Partial<Record<keyof FleetSettings, string>> = {
deployKeepDays: 'DEPLOY_KEEP_DAYS', restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY',
deployKeepDays: 'DEPLOY_KEEP_DAYS',
restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY',
createConcurrency: 'DEPLOYMENT_CREATE_CONCURRENCY'
};
for (const key of Object.keys(legacy) as (keyof FleetSettings)[]) {
const raw = process.env[legacy[key]!];
if (raw) values[key] = Number(raw);
}
handle.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)')
handle
.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)')
.run(FLEET_SETTINGS_KEY, JSON.stringify(validateFleetSettings(values)));
}
+274 -80
View File
@@ -28,9 +28,15 @@ let optimizeTimer: NodeJS.Timeout | undefined;
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
// backfill columns; this can.
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'container_checked_at', ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT' },
{
name: 'container_checked_at',
ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT'
},
{ name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
{ name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
{
name: 'desired_state',
ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))"
},
// Base columns that shipped in the very first schema stay here for parity
// with fresh installs — no ALTER is issued if they're already present.
{ name: 'game_port', ddl: 'ALTER TABLE servers ADD COLUMN game_port INTEGER' },
@@ -43,7 +49,10 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'last_roster', ddl: 'ALTER TABLE servers ADD COLUMN last_roster TEXT' },
{ name: 'last_seen_at', ddl: 'ALTER TABLE servers ADD COLUMN last_seen_at TEXT' },
{ name: 'last_error', ddl: 'ALTER TABLE servers ADD COLUMN last_error TEXT' },
{ name: 'rcon_failures', ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0' },
{
name: 'rcon_failures',
ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0'
},
{ name: 'last_reply_snippet', ddl: 'ALTER TABLE servers ADD COLUMN last_reply_snippet TEXT' },
// Script-scope columns — static per server, populated via server-edit UI.
{ name: 'game', ddl: 'ALTER TABLE servers ADD COLUMN game TEXT' },
@@ -78,7 +87,10 @@ function migrateServers(handle: Database.Database): void {
const retired = new Set(['komodo_type', 'komodo_name', 'komodo_server']);
handle.transaction(() => {
for (const column of rows) {
if (retired.has(column.name)) handle.exec(`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`);
if (retired.has(column.name))
handle.exec(
`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`
);
}
})();
}
@@ -98,8 +110,14 @@ const SCRIPT_VERSIONS_COLUMNS: Array<{ name: string; ddl: string }> = [
// the authoritative "promoted" signal), and any version already gated
// through a ready-promote beta will be re-marked next time that state
// machine fires.
{ name: 'state', ddl: "ALTER TABLE script_versions ADD COLUMN state TEXT NOT NULL DEFAULT 'untested'" },
{ name: 'kind', ddl: "ALTER TABLE script_versions ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" }
{
name: 'state',
ddl: "ALTER TABLE script_versions ADD COLUMN state TEXT NOT NULL DEFAULT 'untested'"
},
{
name: 'kind',
ddl: "ALTER TABLE script_versions ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'"
}
];
const BETA_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [
@@ -107,18 +125,40 @@ const BETA_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [
];
const DEPLOYMENT_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'kind', ddl: "ALTER TABLE deployment_scripts ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" }
{
name: 'kind',
ddl: "ALTER TABLE deployment_scripts ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'"
}
];
const DEPLOYMENTS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'script_id', ddl: 'ALTER TABLE deployments ADD COLUMN script_id INTEGER REFERENCES scripts(id)' },
{ name: 'script_version_id', ddl: 'ALTER TABLE deployments ADD COLUMN script_version_id INTEGER REFERENCES script_versions(id)' },
{ name: 'prior_version_id', ddl: 'ALTER TABLE deployments ADD COLUMN prior_version_id INTEGER REFERENCES script_versions(id)' },
{ name: 'beta_id', ddl: 'ALTER TABLE deployments ADD COLUMN beta_id INTEGER REFERENCES betas(id)' },
{ name: 'rolled_back_by', ddl: 'ALTER TABLE deployments ADD COLUMN rolled_back_by TEXT REFERENCES deployments(id)' }
{
name: 'script_id',
ddl: 'ALTER TABLE deployments ADD COLUMN script_id INTEGER REFERENCES scripts(id)'
},
{
name: 'script_version_id',
ddl: 'ALTER TABLE deployments ADD COLUMN script_version_id INTEGER REFERENCES script_versions(id)'
},
{
name: 'prior_version_id',
ddl: 'ALTER TABLE deployments ADD COLUMN prior_version_id INTEGER REFERENCES script_versions(id)'
},
{
name: 'beta_id',
ddl: 'ALTER TABLE deployments ADD COLUMN beta_id INTEGER REFERENCES betas(id)'
},
{
name: 'rolled_back_by',
ddl: 'ALTER TABLE deployments ADD COLUMN rolled_back_by TEXT REFERENCES deployments(id)'
}
];
function migrateTable(handle: Database.Database, table: string, cols: Array<{ name: string; ddl: string }>): void {
function migrateTable(
handle: Database.Database,
table: string,
cols: Array<{ name: string; ddl: string }>
): void {
const rows = handle.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
const existing = new Set(rows.map((r) => r.name));
for (const col of cols) {
@@ -169,14 +209,16 @@ function backfillMultiScriptJoins(handle: Database.Database): void {
// operator can promote/rollback in the old app before upgrading.
function dropLegacyBetaTables(handle: Database.Database): void {
const legacy = handle
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name IN ('betas','beta_scripts','beta_deploys')")
.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name IN ('betas','beta_scripts','beta_deploys')"
)
.all() as Array<{ name: string }>;
if (legacy.length === 0) return;
const legacyNames = legacy.map((r) => r.name);
// The old `betas` table had a `slug` column; the new one has `id` + `script_id`.
// If we see `slug`, we know it's the old shape.
const betasCols = handle.prepare("PRAGMA table_info(betas)").all() as Array<{ name: string }>;
const betasCols = handle.prepare('PRAGMA table_info(betas)').all() as Array<{ name: string }>;
const isLegacyShape = betasCols.some((c) => c.name === 'slug');
if (!isLegacyShape) return;
@@ -205,8 +247,7 @@ function dropLegacyBetaTables(handle: Database.Database): void {
// active betas aren't blocked.
function assertSnapshotSchemeSafe(handle: Database.Database): void {
const done = handle.prepare("SELECT value FROM settings WHERE key = 'snapshots_v1'").get() as
| { value: string }
| undefined;
{ value: string } | undefined;
if (done) return;
const active = handle
.prepare("SELECT id FROM betas WHERE state IN ('running','ready-promote') LIMIT 1")
@@ -227,19 +268,36 @@ function migrateGameTelemetry(handle: Database.Database): void {
handle.transaction(() => {
const convert = (raw: string): string => {
const plan = JSON.parse(raw) as BetaPlan;
plan.requirements = plan.requirements.map(r => {
plan.requirements = plan.requirements.map((r) => {
const { event: _event, ...rest } = r as BetaRequirement & { event?: string };
return ['feature', 'restart-budget', 'clean-logs'].includes(r.kind)
? { ...requirement('manual', r.id), label: `Review: ${r.label}`.slice(0, 200) } : rest;
? { ...requirement('manual', r.id), label: `Review: ${r.label}`.slice(0, 200) }
: rest;
});
return JSON.stringify(plan);
};
const plans = handle.prepare('SELECT beta_id, plan FROM beta_test_plans').all() as Array<{ beta_id: number; plan: string }>;
for (const row of plans) handle.prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?').run(convert(row.plan), row.beta_id);
const presets = handle.prepare('SELECT name, plan FROM beta_test_presets').all() as Array<{ name: string; plan: string }>;
for (const row of presets) handle.prepare('UPDATE beta_test_presets SET plan = ? WHERE name = ?').run(convert(row.plan), row.name);
handle.prepare(`UPDATE beta_test_runs SET metrics = ?, cursors = '{}' WHERE retired_at IS NULL
AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'))`).run(JSON.stringify(emptyMetrics()));
const plans = handle.prepare('SELECT beta_id, plan FROM beta_test_plans').all() as Array<{
beta_id: number;
plan: string;
}>;
for (const row of plans)
handle
.prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?')
.run(convert(row.plan), row.beta_id);
const presets = handle.prepare('SELECT name, plan FROM beta_test_presets').all() as Array<{
name: string;
plan: string;
}>;
for (const row of presets)
handle
.prepare('UPDATE beta_test_presets SET plan = ? WHERE name = ?')
.run(convert(row.plan), row.name);
handle
.prepare(
`UPDATE beta_test_runs SET metrics = ?, cursors = '{}' WHERE retired_at IS NULL
AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'))`
)
.run(JSON.stringify(emptyMetrics()));
handle.exec(`UPDATE beta_test_plans SET outcome = 'active', reason = 'Gameplay telemetry updated; collecting fresh script events'
WHERE outcome IN ('active','passed') AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'));
UPDATE script_versions SET state = 'untested' WHERE state = 'tested'
@@ -254,54 +312,96 @@ function migrateGameTelemetry(handle: Database.Database): void {
}
function migrateProvisioningActions(handle: Database.Database): void {
const table = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'").get() as { sql: string };
const table = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'"
)
.get() as { sql: string };
if (table.sql.includes("'rebuild'")) return;
// Rebuild the CHECK constraint without cascading into events, recipes, or
// collection members. Preserve rowids: they order same-server operations.
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(table.sql.replace('server_deployments', 'server_deployments_new')
.replace("'create', 'destroy'", "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"));
const columns = (handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[])
.map(column => `"${column.name.replaceAll('"', '""')}"`).join(', ');
handle.exec(
table.sql
.replace('server_deployments', 'server_deployments_new')
.replace(
"'create', 'destroy'",
"'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"
)
);
const columns = (
handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[]
)
.map((column) => `"${column.name.replaceAll('"', '""')}"`)
.join(', ');
handle.exec(`INSERT INTO server_deployments_new (rowid, ${columns})
SELECT rowid, ${columns} FROM server_deployments;
DROP TABLE server_deployments;
ALTER TABLE server_deployments_new RENAME TO server_deployments;`);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Provisioning migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Provisioning migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
/** Rebuild constrained tables while preserving links, indexes, triggers, and row order. */
function migrateWatchdogConstraints(handle: Database.Database): void {
for (const name of ['provisioning_sources', 'deployment_servers']) {
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
const replacement = name === 'provisioning_sources'
? (sql.includes("'autoscaling'") ? sql : sql.replace(/'campaign','beta','maintenance'(?:,'watchdog')?/, "'campaign','beta','maintenance','watchdog','autoscaling'"))
: sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
.get(name) as { sql: string };
const replacement =
name === 'provisioning_sources'
? sql.includes("'autoscaling'")
? sql
: sql.replace(
/'campaign','beta','maintenance'(?:,'watchdog')?/,
"'campaign','beta','maintenance','watchdog','autoscaling'"
)
: sql.replace(
/server\s+TEXT NOT NULL REFERENCES servers\(name\)/,
'server TEXT NOT NULL'
);
if (sql === replacement) continue;
const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependent = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all(name) as { sql: string }[];
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement.replace(name, name + '_watchdog_new'));
handle.exec(`INSERT INTO ${name}_watchdog_new (rowid, ${
(handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
}) SELECT rowid, ${
(handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
} FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};`);
handle.exec(
`INSERT INTO ${name}_watchdog_new (rowid, ${(
handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]
)
.map((col) => col.name)
.join(', ')}) SELECT rowid, ${(
handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]
)
.map((col) => col.name)
.join(
', '
)} FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};`
);
for (const object of dependent) handle.exec(object.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Watchdog migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Watchdog migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
}
function backfillDesiredState(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) return;
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get())
return;
handle.transaction(() => {
// A failed stop may have reached the host agent. Preserve that explicit intent.
handle.exec(`UPDATE servers SET desired_state = 'stopped' WHERE 'stop' = (
@@ -331,35 +431,80 @@ export function db(): Database.Database {
migrateProvisioningActions(handle);
migrateWatchdogConstraints(handle);
migrateTable(handle, 'deployment_files', [
{ name: 'snapshot_pending', ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0' }
{
name: 'snapshot_pending',
ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0'
}
]);
migrateTable(handle, 'restart_rollout', [
{
name: 'operation_id',
ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT'
}
]);
migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
migrateTable(handle, 'server_deployments', [
{ name: 'requested_by', ddl: 'ALTER TABLE server_deployments ADD COLUMN requested_by TEXT' },
{ name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
{ name: 'checked_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT' },
{
name: 'requested_by',
ddl: 'ALTER TABLE server_deployments ADD COLUMN requested_by TEXT'
},
{
name: 'submitted_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT'
},
{
name: 'checked_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT'
},
{ name: 'players', ddl: 'ALTER TABLE server_deployments ADD COLUMN players INTEGER' },
{ name: 'phase_key', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_key TEXT' },
{ name: 'phase_started_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT' },
{ name: 'observed_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT' },
{ name: 'observation_json', ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT' }
{
name: 'phase_started_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT'
},
{
name: 'observed_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT'
},
{
name: 'observation_json',
ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT'
}
]);
migrateTable(handle, 'server_deployment_events', [
{ name: 'kind', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN kind TEXT' },
{ name: 'phase_key', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT' },
{ name: 'details_json', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT' }
{
name: 'phase_key',
ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT'
},
{
name: 'details_json',
ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT'
}
]);
for (const table of ['deployment_servers', 'beta_servers', 'maintenance_jobs']) {
migrateTable(handle, table, [{ name: 'provisioning_id', ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` }]);
migrateTable(handle, table, [
{
name: 'provisioning_id',
ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT`
}
]);
}
migrateTable(handle, 'maintenance_jobs', [
{ name: 'kind', ddl: "ALTER TABLE maintenance_jobs ADD COLUMN kind TEXT NOT NULL DEFAULT 'legacy'" },
{ name: 'origin', ddl: "ALTER TABLE maintenance_jobs ADD COLUMN origin TEXT NOT NULL DEFAULT 'manual'" }
{
name: 'kind',
ddl: "ALTER TABLE maintenance_jobs ADD COLUMN kind TEXT NOT NULL DEFAULT 'legacy'"
},
{
name: 'origin',
ddl: "ALTER TABLE maintenance_jobs ADD COLUMN origin TEXT NOT NULL DEFAULT 'manual'"
}
]);
migrateServers(handle);
// Old snippets may contain player addresses, including hex-encoded bytes.
// Discard them once; newly retained diagnostics are sanitized failure replies only.
if (!handle.prepare("SELECT 1 FROM settings WHERE key='rcon_private_diagnostics_v1'").get()) {
if (
!handle.prepare("SELECT 1 FROM settings WHERE key='rcon_private_diagnostics_v1'").get()
) {
handle.transaction(() => {
handle.exec(`UPDATE servers SET last_reply_snippet=NULL;
UPDATE servers SET last_error='bad-reply:unparseable print reply'
@@ -374,25 +519,52 @@ export function db(): Database.Database {
migrateTable(handle, 'scripts', SCRIPTS_COLUMNS);
migrateTable(handle, 'script_versions', SCRIPT_VERSIONS_COLUMNS);
migrateTable(handle, 'beta_scripts', BETA_SCRIPTS_COLUMNS);
migrateTable(handle, 'beta_test_runs', [{ name: 'revision', ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1' }]);
migrateTable(handle, 'beta_test_runs', [
{
name: 'revision',
ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1'
}
]);
migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS);
migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS);
// Older databases gain rolled_back_by above, after schema.sql runs.
handle.exec('CREATE INDEX IF NOT EXISTS idx_deployments_rolled_back_by ON deployments(rolled_back_by)');
handle.exec(
'CREATE INDEX IF NOT EXISTS idx_deployments_rolled_back_by ON deployments(rolled_back_by)'
);
migrateTable(handle, 'server_launch_collections', [
{ name: 'cancelled_at', ddl: 'ALTER TABLE server_launch_collections ADD COLUMN cancelled_at TEXT' },
{ name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" }
{
name: 'cancelled_at',
ddl: 'ALTER TABLE server_launch_collections ADD COLUMN cancelled_at TEXT'
},
{
name: 'mode',
ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'"
},
{
name: 'retained_servers',
ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'"
}
]);
migrateTable(handle, 'server_launch_recipes', [
{ name: 'recipe_summary', ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT' }
{
name: 'recipe_summary',
ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT'
}
]);
migrateServerHistory(handle);
migrateTable(handle, 'server_collection_previews', [
{ name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'fleet_snapshot', ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT' }
{
name: 'mode',
ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'"
},
{
name: 'fleet_snapshot',
ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT'
}
]);
migrateTable(handle, 'server_telemetry', [
{ name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' }
]);
migrateTable(handle, 'server_telemetry', [{ name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' }]);
// Kind-referencing indexes: safe here because migrateTable above just
// ALTER'd the column into existence if it was missing.
handle.exec(
@@ -403,14 +575,27 @@ export function db(): Database.Database {
backfillMultiScriptJoins(handle);
assertSnapshotSchemeSafe(handle);
migrateGameTelemetry(handle);
migrateTable(handle, 'leaderboard_matches', [{ name: 'start_players', ddl: 'ALTER TABLE leaderboard_matches ADD COLUMN start_players INTEGER' }]);
migrateTable(handle, 'leaderboard_matches', [
{
name: 'start_players',
ddl: 'ALTER TABLE leaderboard_matches ADD COLUMN start_players INTEGER'
}
]);
migrateTable(handle, 'leaderboard_match_players', [
{ name: 'first_observed_round', ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN first_observed_round INTEGER' },
{ name: 'present_at_start', ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN present_at_start INTEGER' }
{
name: 'first_observed_round',
ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN first_observed_round INTEGER'
},
{
name: 'present_at_start',
ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN present_at_start INTEGER'
}
]);
backfillLeaderboard(handle);
initializeFleetSettings(handle);
migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
migrationStep('native instance schema and file plans', () =>
migrateNativeInstances(handle)
);
migrationStep('per-game server key pools', () => migrateServerKeys(handle));
migrationStep('autoscaling schema', () => ensureAutoscalingSchema(handle));
migrationStep('rebalancing schema', () => ensureRebalancingSchema(handle));
@@ -436,12 +621,18 @@ export function db(): Database.Database {
ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`);
installPerformanceIndexes(handle);
installReportRevisions(handle);
migrationStep('retention clocks and indexes', () => retentionClocks(handle, {
deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'],
maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'],
leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at']
}));
migrationStep('retention clocks and indexes', () =>
retentionClocks(handle, {
deployments: ['started_at', 'finished_at'],
server_deployments: ['started_at', 'finished_at'],
maintenance_jobs: ['requested_at', 'finished_at'],
watchdog_incidents: ['first_seen_at', 'finished_at'],
leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
betas: ['opened_at', 'closed_at'],
script_versions: ['uploaded_at'],
retention_controls: ['resolved_at']
})
);
// Existing databases require one rebuild to gain the incremental-vacuum page map.
enableIncrementalVacuum(handle);
handle.pragma('optimize = 0x10002');
@@ -472,8 +663,11 @@ export function db(): Database.Database {
_db = handle;
optimizeTimer = setInterval(() => {
try { handle.pragma('optimize'); }
catch (error) { console.error('[database] optimize failed:', error); }
try {
handle.pragma('optimize');
} catch (error) {
console.error('[database] optimize failed:', error);
}
}, 60 * 60_000);
optimizeTimer.unref();
return _db;
+53 -15
View File
@@ -3,30 +3,59 @@ import { migrationStep, tuneMigrationConnection } from './connection.js';
/** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */
export function retentionClocks(handle: Database.Database, clocks: Record<string, string[]>): void {
handle.exec('CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)');
handle.exec(
'CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)'
);
for (const [table, columns] of Object.entries(clocks)) {
const marker = `iso-clocks:${table}:v1`;
if (!handle.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?').get(marker)) {
if (
!handle
.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?')
.get(marker)
) {
handle.transaction(() => {
for (const column of columns) handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column})
for (const column of columns)
handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column})
WHERE julianday(${column}) IS NOT NULL AND ${column}!=strftime('%Y-%m-%dT%H:%M:%fZ',${column});`);
handle.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)').run(marker);
handle
.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)')
.run(marker);
})();
}
const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as { name: string; partial: number }[];
const indexed = new Set(indexes.filter(index => !index.partial).map(index =>
(handle.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`).get() as { name: string }).name));
for (const column of columns) if (!indexed.has(column)) {
handle.exec(`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`);
}
const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as {
name: string;
partial: number;
}[];
const indexed = new Set(
indexes
.filter((index) => !index.partial)
.map(
(index) =>
(
handle
.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`)
.get() as { name: string }
).name
)
);
for (const column of columns)
if (!indexed.has(column)) {
handle.exec(
`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`
);
}
}
}
/** VACUUM can renumber implicit rowids. Keep host enable cutoffs tied to command IDs across crashes. */
export function enableIncrementalVacuum(handle: Database.Database): void {
if (handle.pragma('auto_vacuum', { simple: true }) !== 2) {
handle.exec(`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`);
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) {
handle.exec(
`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`
);
if (
handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()
) {
handle.exec(`INSERT OR IGNORE INTO database_vacuum_boundaries
SELECT e.host_id,(SELECT o.id FROM ec_host_operations o WHERE o.host_id=e.host_id
AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`);
@@ -42,7 +71,9 @@ export function enableIncrementalVacuum(handle: Database.Database): void {
};
try {
tuneMigrationConnection(handle);
migrationStep('incremental vacuum page map (disk-backed rebuild)', () => handle.exec('VACUUM'));
migrationStep('incremental vacuum page map (disk-backed rebuild)', () =>
handle.exec('VACUUM')
);
} finally {
handle.pragma('shrink_memory');
handle.pragma(`temp_store = ${previous.temp}`);
@@ -51,9 +82,16 @@ export function enableIncrementalVacuum(handle: Database.Database): void {
handle.pragma(`journal_size_limit = ${previous.journal}`);
}
}
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) {
if (
handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()
) {
handle.transaction(() => {
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) handle.exec(`
if (
handle
.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'")
.get()
)
handle.exec(`
UPDATE ec_host_enable_requests AS e SET operation_floor=COALESCE((
SELECT o.rowid FROM database_vacuum_boundaries b JOIN ec_host_operations o ON o.id=b.operation_id
WHERE b.host_id=e.host_id),0) WHERE e.host_id IN (SELECT host_id FROM database_vacuum_boundaries)`);
+47 -18
View File
@@ -1,17 +1,34 @@
import type Database from 'better-sqlite3';
const historyTables = ['beta_servers', 'deployment_servers', 'server_deployments', 'leaderboard_matches', 'watchdog_incidents', 'maintenance_jobs'];
const historyTables = [
'beta_servers',
'deployment_servers',
'server_deployments',
'leaderboard_matches',
'watchdog_incidents',
'maintenance_jobs'
];
const now = "strftime('%Y-%m-%dT%H:%M:%fZ','now')";
/** Detach durable beta records without dropping their evidence, indexes, or activity triggers. */
function detachBetaHistory(handle: Database.Database): void {
for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) {
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
const replacement = sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
.get(name) as { sql: string };
const replacement = sql.replace(
/server\s+TEXT NOT NULL REFERENCES servers\(name\)/,
'server TEXT NOT NULL'
);
if (replacement === sql) continue;
const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependents = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all(name) as { sql: string }[];
const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ');
const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[])
.map((col) => col.name)
.join(', ');
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
@@ -19,9 +36,12 @@ function detachBetaHistory(handle: Database.Database): void {
handle.exec(`INSERT INTO ${name}_history_new (rowid, ${columns}) SELECT rowid, ${columns} FROM ${name};
DROP TABLE ${name}; ALTER TABLE ${name}_history_new RENAME TO ${name};`);
for (const dependent of dependents) handle.exec(dependent.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Server history migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Server history migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
}
@@ -40,12 +60,19 @@ export function migrateServerHistory(handle: Database.Database): void {
detachBetaHistory(handle);
handle.transaction(() => {
for (const table of historyTables) {
const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[];
if (!columns.some(column => column.name === 'server_deleted_at')) handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`);
const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as {
name: string;
}[];
if (!columns.some((column) => column.name === 'server_deleted_at'))
handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`);
}
const collectionColumns = handle.prepare('PRAGMA table_info(server_launch_collections)').all() as { name: string }[];
if (!collectionColumns.some(column => column.name === 'retained_server_deletions')) {
handle.exec("ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'");
const collectionColumns = handle
.prepare('PRAGMA table_info(server_launch_collections)')
.all() as { name: string }[];
if (!collectionColumns.some((column) => column.name === 'retained_server_deletions')) {
handle.exec(
"ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'"
);
}
// Historical unfinished work cannot claim the next server with this name.
handle.exec(`DROP INDEX IF EXISTS idx_maintenance_pending;
@@ -96,15 +123,17 @@ export function migrateServerHistory(handle: Database.Database): void {
OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`;
for (const table of historyTables) {
const exception = table === 'server_deployments'
? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS (
const exception =
table === 'server_deployments'
? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS (
SELECT 1 FROM server_deployments removal WHERE removal.server = server_deployments.server
AND removal.rowid > server_deployments.rowid AND removal.action = 'destroy' AND removal.state IN ('done','warning')))
AND id NOT IN (${liveRebuilds})`
: table === 'deployment_servers'
? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))`
: table === 'watchdog_incidents'
? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))` : '';
: table === 'deployment_servers'
? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))`
: table === 'watchdog_incidents'
? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))`
: '';
handle.exec(`UPDATE ${table} SET server_deleted_at = ${now} WHERE server_deleted_at IS NULL
AND NOT EXISTS (SELECT 1 FROM servers WHERE name = ${table}.server AND destroyed_at IS NULL) ${exception}`);
}
+116 -64
View File
@@ -22,79 +22,131 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_server_key_leases_current ON server_key_le
/** Preserve every occupied legacy assignment, even duplicated credentials. */
export function migrateServerKeys(handle: Database.Database): void {
handle.transaction(() => {
const legacy = (handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[])
.some(column => column.name === 'server_id');
if (legacy) handle.exec('ALTER TABLE server_keys RENAME TO legacy_server_keys');
handle.exec(keySchema + leaseSchema);
if (!legacy) return;
const rows = handle.prepare(`SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id`).all() as
{ server_id: string; pluto_key: string; updated_at: string }[];
const occupied = handle.prepare(`SELECT 1 WHERE
handle
.transaction(() => {
const legacy = (
handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[]
).some((column) => column.name === 'server_id');
if (legacy) handle.exec('ALTER TABLE server_keys RENAME TO legacy_server_keys');
handle.exec(keySchema + leaseSchema);
if (!legacy) return;
const rows = handle
.prepare(
`SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id`
)
.all() as { server_id: string; pluto_key: string; updated_at: string }[];
const occupied = handle.prepare(`SELECT 1 WHERE
EXISTS (SELECT 1 FROM servers WHERE name=@server AND destroyed_at IS NULL)
OR EXISTS (SELECT 1 FROM native_instances WHERE server=@server AND retired_at IS NULL)
OR EXISTS (SELECT 1 FROM server_deployments WHERE server=@server AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state NOT IN ('done','warning','cancelled'))`);
const find = handle.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?');
const serverGame = handle.prepare("SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')");
const insert = handle.prepare('INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)');
const lease = handle.prepare('INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)');
const assignments = new Map<string, string[]>();
function save(game: string, credential: string, label: string, updatedAt: string): string {
const existing = (find.get(game, credential) as { id: string } | undefined)?.id;
if (existing) return existing;
const id = randomUUID();
insert.run(id, game, label, credential, updatedAt);
return id;
}
for (const row of rows) {
const game = /^(t5|t6)_/.exec(row.server_id)?.[1]
?? (serverGame.get(row.server_id) as { game: string } | undefined)?.game;
if (!game) throw new Error(`Cannot determine the game for saved server key ${row.server_id}`);
const id = save(game, row.pluto_key, row.server_id, row.updated_at);
if (occupied.get({ server: row.server_id })) assignments.set(row.server_id, [id]);
}
// The old form could overwrite a saved key without changing the running
// container. Durable create/rebuild payloads are stronger evidence of
// its actual credential. Keep ambiguous delivered attempts as additional
// holds until the instance is destroyed; never free a possibly live key.
const hasTable = (name: string) => !!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name);
if (hasTable('ec_host_operations')) {
const payload = hasTable('ec_host_operation_payloads')
? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)' : 'o.payload';
const command = `CASE WHEN json_valid(${payload}) THEN ${payload} ELSE '{}' END`;
const launched = handle.prepare(`SELECT n.server,o.state,o.updated_at,
const find = handle.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?');
const serverGame = handle.prepare(
"SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')"
);
const insert = handle.prepare(
'INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)'
);
const lease = handle.prepare(
'INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)'
);
const assignments = new Map<string, string[]>();
function save(
game: string,
credential: string,
label: string,
updatedAt: string
): string {
const existing = (find.get(game, credential) as { id: string } | undefined)?.id;
if (existing) return existing;
const id = randomUUID();
insert.run(id, game, label, credential, updatedAt);
return id;
}
for (const row of rows) {
const game =
/^(t5|t6)_/.exec(row.server_id)?.[1] ??
(serverGame.get(row.server_id) as { game: string } | undefined)?.game;
if (!game)
throw new Error(
`Cannot determine the game for saved server key ${row.server_id}`
);
const id = save(game, row.pluto_key, row.server_id, row.updated_at);
if (occupied.get({ server: row.server_id })) assignments.set(row.server_id, [id]);
}
// The old form could overwrite a saved key without changing the running
// container. Durable create/rebuild payloads are stronger evidence of
// its actual credential. Keep ambiguous delivered attempts as additional
// holds until the instance is destroyed; never free a possibly live key.
const hasTable = (name: string) =>
!!handle
.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?")
.get(name);
if (hasTable('ec_host_operations')) {
const payload = hasTable('ec_host_operation_payloads')
? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'
: 'o.payload';
const command = `CASE WHEN json_valid(${payload}) THEN ${payload} ELSE '{}' END`;
const launched = handle
.prepare(
`SELECT n.server,o.state,o.updated_at,
json_extract(${command},'$.env.PLUTO_SERVER_KEY') AS credential,
json_extract(${command},'$.game') AS game,
json_extract(${command},'$.env.PLUTAINER_GAME') AS game_id
FROM native_instances n JOIN ec_host_operations o ON o.instance_id=n.id AND o.host_id=n.host_id
WHERE n.retired_at IS NULL AND o.kind IN ('instance.create','instance.rebuild') AND o.state!='failed'
AND (o.generation=n.generation OR json_extract(${command},'$.nextGeneration')=n.generation)
ORDER BY o.rowid DESC`).all() as
{ server: string; state: string; updated_at: string; credential: unknown; game: unknown; game_id: unknown }[];
const recovered = new Map<string, string[]>();
const complete = new Set<string>();
for (const row of launched) {
if (complete.has(row.server) || typeof row.credential !== 'string' || !row.credential) continue;
const game = row.game === 't5' || row.game === 't6' ? row.game
: row.game_id === 't5sp' ? 't5' : row.game_id === 't6zm' ? 't6'
: /^(t5|t6)_/.exec(row.server)?.[1]
?? (serverGame.get(row.server) as { game: string } | undefined)?.game;
if (!game) throw new Error(`Cannot determine the game for launched server ${row.server}`);
const ids = recovered.get(row.server) ?? [];
const id = save(game, row.credential, row.server, row.updated_at);
if (!ids.includes(id)) ids.push(id);
recovered.set(row.server, ids);
if (row.state === 'completed') complete.add(row.server);
ORDER BY o.rowid DESC`
)
.all() as {
server: string;
state: string;
updated_at: string;
credential: unknown;
game: unknown;
game_id: unknown;
}[];
const recovered = new Map<string, string[]>();
const complete = new Set<string>();
for (const row of launched) {
if (
complete.has(row.server) ||
typeof row.credential !== 'string' ||
!row.credential
)
continue;
const game =
row.game === 't5' || row.game === 't6'
? row.game
: row.game_id === 't5sp'
? 't5'
: row.game_id === 't6zm'
? 't6'
: (/^(t5|t6)_/.exec(row.server)?.[1] ??
(serverGame.get(row.server) as { game: string } | undefined)
?.game);
if (!game)
throw new Error(
`Cannot determine the game for launched server ${row.server}`
);
const ids = recovered.get(row.server) ?? [];
const id = save(game, row.credential, row.server, row.updated_at);
if (!ids.includes(id)) ids.push(id);
recovered.set(row.server, ids);
if (row.state === 'completed') complete.add(row.server);
}
for (const [server, ids] of recovered) {
// If no completed launch receipt survives, its saved assignment
// remains a possible prior credential while the attempt resolves.
if (!complete.has(server))
for (const id of assignments.get(server) ?? [])
if (!ids.includes(id)) ids.push(id);
assignments.set(server, ids);
}
}
for (const [server, ids] of recovered) {
// If no completed launch receipt survives, its saved assignment
// remains a possible prior credential while the attempt resolves.
if (!complete.has(server)) for (const id of assignments.get(server) ?? []) if (!ids.includes(id)) ids.push(id);
assignments.set(server, ids);
}
}
for (const [server, ids] of assignments) ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0));
handle.exec('DROP TABLE legacy_server_keys');
}).immediate();
for (const [server, ids] of assignments)
ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0));
handle.exec('DROP TABLE legacy_server_keys');
})
.immediate();
}
+21 -9
View File
@@ -1,19 +1,31 @@
import type Database from 'better-sqlite3';
/** Log query shapes, never bound values or SQL string literals. Disabled by default. */
export function timeStatement(statement: Database.Statement, sql: string, thresholdMs: number): Database.Statement {
export function timeStatement(
statement: Database.Statement,
sql: string,
thresholdMs: number
): Database.Statement {
if (!Number.isFinite(thresholdMs) || thresholdMs <= 0) return statement;
const shape = sql.replace(/'(?:''|[^'])*'/g, '?').replace(/\s+/g, ' ').trim().slice(0, 200);
const shape = sql
.replace(/'(?:''|[^'])*'/g, '?')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 200);
for (const method of ['run', 'get', 'all'] as const) {
const original = statement[method].bind(statement) as (...args: unknown[]) => unknown;
Object.defineProperty(statement, method, { value: (...args: unknown[]) => {
const started = performance.now();
try { return original(...args); }
finally {
const ms = performance.now() - started;
if (ms >= thresholdMs) console.warn('[sql]', { ms: Math.round(ms), method, sql: shape });
Object.defineProperty(statement, method, {
value: (...args: unknown[]) => {
const started = performance.now();
try {
return original(...args);
} finally {
const ms = performance.now() - started;
if (ms >= thresholdMs)
console.warn('[sql]', { ms: Math.round(ms), method, sql: shape });
}
}
} });
});
}
return statement;
}
+27 -9
View File
@@ -3,25 +3,43 @@ import type Database from 'better-sqlite3';
/** Keep metadata ahead of overflow pages, preserving IDs and all references. */
export function migrateVersionLayout(handle: Database.Database): void {
const columns = handle.pragma('table_info(script_versions)') as { name: string }[];
if (!columns.some(column => column.name === 'content') || columns.at(-1)?.name === 'content') return;
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'").get() as { sql: string };
if (!columns.some((column) => column.name === 'content') || columns.at(-1)?.name === 'content')
return;
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'")
.get() as { sql: string };
const content = /\bcontent\s+BLOB\s+NOT\s+NULL\s*,/i;
if (!content.test(sql)) throw new Error('Unrecognized script version content layout');
const replacement = sql.replace(content, '').replace(/,\s*UNIQUE\s*\(/i, ', content BLOB NOT NULL, UNIQUE(')
const replacement = sql
.replace(content, '')
.replace(/,\s*UNIQUE\s*\(/i, ', content BLOB NOT NULL, UNIQUE(')
.replace('script_versions', 'script_versions_metadata');
const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='script_versions' AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependents = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name='script_versions' AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all() as { sql: string }[];
const sequence = handle.prepare("SELECT seq FROM sqlite_sequence WHERE name='script_versions'").get() as { seq: number } | undefined;
const sequence = handle
.prepare("SELECT seq FROM sqlite_sequence WHERE name='script_versions'")
.get() as { seq: number } | undefined;
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement);
const names = columns.map(column => `"${column.name}"`).join(',');
const names = columns.map((column) => `"${column.name}"`).join(',');
handle.exec(`INSERT INTO script_versions_metadata (${names}) SELECT ${names} FROM script_versions;
DROP TABLE script_versions; ALTER TABLE script_versions_metadata RENAME TO script_versions;`);
if (sequence) handle.prepare("UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name='script_versions'").run(sequence.seq);
if (sequence)
handle
.prepare(
"UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name='script_versions'"
)
.run(sequence.seq);
for (const dependent of dependents) handle.exec(dependent.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Version layout migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Version layout migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
+81 -26
View File
@@ -19,14 +19,30 @@ export interface TestMetrics {
outages: number;
streaks: Record<string, number>;
}
export const emptyMetrics = (): TestMetrics => ({ values: {}, manual: {}, outages: 0, streaks: {} });
export const emptyMetrics = (): TestMetrics => ({
values: {},
manual: {},
outages: 0,
streaks: {}
});
/** Gameplay comes only from the script. No reporter IDs or beta handshake. */
export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: GameEvent, now: number): void {
export function applyGameEvent(
plan: BetaPlan,
metrics: TestMetrics,
event: GameEvent,
now: number
): void {
const players = event.players.length;
if (event.event === 'game_start' || !metrics.match || metrics.match.map !== event.map) {
metrics.match = { map: event.map, started: event.event === 'game_start' ? now : undefined,
lastEvent: now, players, ended: false, rounds: {} };
metrics.match = {
map: event.map,
started: event.event === 'game_start' ? now : undefined,
lastEvent: now,
players,
ended: false,
rounds: {}
};
metrics.streaks = {};
}
const match = metrics.match;
@@ -34,10 +50,12 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game
const minutes = Math.max(0, now - match.lastEvent) / 60_000;
for (const r of plan.requirements) {
const current = metrics.values[r.id] ?? 0;
const populated = players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers);
const populated =
players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers);
if (r.kind === 'population') metrics.values[r.id] = Math.max(current, players);
if (r.kind === 'populated-minutes' && populated) metrics.values[r.id] = current + minutes;
if (r.kind === 'player-hours' && populated) metrics.values[r.id] = current + minutes * Math.min(players, match.players) / 60;
if (r.kind === 'player-hours' && populated)
metrics.values[r.id] = current + (minutes * Math.min(players, match.players)) / 60;
if (r.kind === 'sustained-population') {
metrics.streaks[r.id] = populated ? (metrics.streaks[r.id] ?? 0) + minutes : 0;
metrics.values[r.id] = Math.max(current, metrics.streaks[r.id]);
@@ -46,10 +64,14 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game
match.rounds[r.id] = Math.max(match.rounds[r.id] ?? 0, event.round);
if (r.kind === 'round') metrics.values[r.id] = Math.max(current, event.round);
}
if (event.event === 'game_end' && r.kind === 'matches'
&& event.round >= MIN_RECORDED_ROUND
&& (match.rounds[r.id] ?? -1) >= r.minRound
&& (r.minMinutes === 0 || (match.started !== undefined && now - match.started >= r.minMinutes * 60_000))) {
if (
event.event === 'game_end' &&
r.kind === 'matches' &&
event.round >= MIN_RECORDED_ROUND &&
(match.rounds[r.id] ?? -1) >= r.minRound &&
(r.minMinutes === 0 ||
(match.started !== undefined && now - match.started >= r.minMinutes * 60_000))
) {
metrics.values[r.id] = current + 1;
}
}
@@ -66,30 +88,63 @@ export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy:
const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
if (!healthy && m.wasHealthy === true) m.outages++;
for (const r of plan.requirements) {
if (r.kind === 'healthy-minutes') m.values[r.id] = healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0;
if (r.kind === 'healthy-minutes')
m.values[r.id] =
healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0;
}
m.lastSample = at; m.wasHealthy = healthy;
m.lastSample = at;
m.wasHealthy = healthy;
}
export function evaluateRequirements(plan: BetaPlan, metrics: TestMetrics, phase: number): RequirementProgress[] {
return plan.requirements.map(r => {
export function evaluateRequirements(
plan: BetaPlan,
metrics: TestMetrics,
phase: number
): RequirementProgress[] {
return plan.requirements.map((r) => {
const budget = r.kind === 'failure-budget';
const current = budget ? metrics.outages : r.kind === 'manual' ? Number(!!metrics.manual[r.id])
: r.kind === 'recovery' ? phase : metrics.values[r.id] ?? 0;
return { id: r.id, label: r.label, current, target: r.target, passed: budget ? current <= r.target : current >= r.target,
const current = budget
? metrics.outages
: r.kind === 'manual'
? Number(!!metrics.manual[r.id])
: r.kind === 'recovery'
? phase
: (metrics.values[r.id] ?? 0);
return {
id: r.id,
label: r.label,
current,
target: r.target,
passed: budget ? current <= r.target : current >= r.target,
failed: budget && current > r.target,
detail: r.kind === 'manual' ? metrics.manual[r.id] ?? 'Awaiting tester sign-off'
: budget ? `${current} / ${r.target} allowed; monitored until promotion`
: `${Math.round(current * 100) / 100} / ${r.target}` };
detail:
r.kind === 'manual'
? (metrics.manual[r.id] ?? 'Awaiting tester sign-off')
: budget
? `${current} / ${r.target} allowed; monitored until promotion`
: `${Math.round(current * 100) / 100} / ${r.target}`
};
});
}
export function coveragePassed(plan: BetaPlan, servers: Array<{ passed: boolean; game: string | null; map: string | null }>): boolean {
export function coveragePassed(
plan: BetaPlan,
servers: Array<{ passed: boolean; game: string | null; map: string | null }>
): boolean {
if (!servers.length) return false;
if (plan.scope === 'all') return servers.every(s => s.passed);
if (plan.scope === 'quorum') return servers.filter(s => s.passed).length >= plan.quorum;
const key = (s: typeof servers[number]) => plan.scope === 'game' ? s.game : plan.scope === 'map' ? s.map
: s.game && s.map ? `${s.game}/${s.map}` : null;
if (plan.scope === 'all') return servers.every((s) => s.passed);
if (plan.scope === 'quorum') return servers.filter((s) => s.passed).length >= plan.quorum;
const key = (s: (typeof servers)[number]) =>
plan.scope === 'game'
? s.game
: plan.scope === 'map'
? s.map
: s.game && s.map
? `${s.game}/${s.map}`
: null;
const groups = new Set(servers.map(key));
return !groups.has(null) && [...groups].every(group => servers.some(s => s.passed && key(s) === group));
return (
!groups.has(null) &&
[...groups].every((group) => servers.some((s) => s.passed && key(s) === group))
);
}
+39 -12
View File
@@ -8,13 +8,21 @@ const positive = (value: string | null, fallback: number) => {
};
export function betaListPage(params: URLSearchParams) {
const state = params.get('state'), view = params.get('view'), search = (params.get('q') ?? '').slice(0, 200).trim();
const predicates: string[] = [], args: (string | number)[] = [];
if (state) { predicates.push('b.state=?'); args.push(state); }
else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')");
const state = params.get('state'),
view = params.get('view'),
search = (params.get('q') ?? '').slice(0, 200).trim();
const predicates: string[] = [],
args: (string | number)[] = [];
if (state) {
predicates.push('b.state=?');
args.push(state);
} else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')");
else if (view === 'history') predicates.push("b.state IN ('promoted','rolled-back')");
const status = params.get('status');
if (status && ['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status)) {
if (
status &&
['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status)
) {
// Match betaStatus: closed lifecycle wins over the final test outcome.
predicates.push(`(CASE WHEN b.state IN ('promoted','rolled-back') THEN b.state
WHEN EXISTS (SELECT 1 FROM beta_servers m WHERE m.beta_id=b.id AND (m.server_deleted_at IS NOT NULL OR m.stage='failed'))
@@ -24,8 +32,15 @@ export function betaListPage(params: URLSearchParams) {
args.push(status);
}
if (search) {
const names = (db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[])
.filter(row => `${row.server} ${serverTitle(row.server)}`.toLowerCase().includes(search.toLowerCase())).map(row => row.server);
const names = (
db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[]
)
.filter((row) =>
`${row.server} ${serverTitle(row.server)}`
.toLowerCase()
.includes(search.toLowerCase())
)
.map((row) => row.server);
predicates.push(`(instr(lower('#'||b.id||' '||COALESCE(b.notes,'')),lower(?))>0 OR EXISTS (
SELECT 1 FROM beta_scripts bs JOIN scripts s ON s.id=bs.script_id WHERE bs.beta_id=b.id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0))
OR EXISTS (SELECT 1 FROM scripts s WHERE s.id=b.script_id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0))
@@ -33,21 +48,33 @@ export function betaListPage(params: URLSearchParams) {
args.push(search, search, search, search, search, JSON.stringify(names));
}
const where = predicates.length ? ' WHERE ' + predicates.join(' AND ') : '';
const total = (db().prepare('SELECT COUNT(*) n FROM betas b' + where).get(...args) as { n: number }).n;
const total = (
db()
.prepare('SELECT COUNT(*) n FROM betas b' + where)
.get(...args) as { n: number }
).n;
const pageSize = Math.min(100, positive(params.get('limit'), 25));
const totalPages = Math.max(1, Math.ceil(total / pageSize));
const page = Math.min(totalPages, positive(params.get('page'), 1));
const betas = db().prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?')
const betas = db()
.prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?')
.all(...args, pageSize, (page - 1) * pageSize) as BetaRow[];
const counts = db().prepare(`SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active,
COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas`).get() as { active: number; history: number };
const counts = db()
.prepare(
`SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active,
COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas`
)
.get() as { active: number; history: number };
return { betas, page, pageSize, total, totalPages, counts };
}
export function betaActivityPage(id: number, params = new URLSearchParams()) {
const before = positive(params.get('before'), Number.MAX_SAFE_INTEGER);
const limit = Math.min(100, positive(params.get('limit'), 50));
const rows = db().prepare('SELECT id,created_at,message FROM beta_events WHERE beta_id=? AND id<? ORDER BY id DESC LIMIT ?')
const rows = db()
.prepare(
'SELECT id,created_at,message FROM beta_events WHERE beta_id=? AND id<? ORDER BY id DESC LIMIT ?'
)
.all(id, before, limit + 1) as { id: number; created_at: string; message: string }[];
const more = rows.length > limit;
const events = rows.slice(0, limit).reverse();
+51 -13
View File
@@ -2,29 +2,67 @@ import { db } from '../db/index.js';
import type { BetaRevision } from '../../beta-revision.js';
export function currentBetaRevision(id: number): { revision: number; started_at: string } {
return db().prepare(`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`).get(id) as
{ revision: number; started_at: string } | undefined
?? { revision: 1, started_at: (db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as { opened_at: string }).opened_at };
return (
(db()
.prepare(
`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`
)
.get(id) as { revision: number; started_at: string } | undefined) ?? {
revision: 1,
started_at: (
db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as {
opened_at: string;
}
).opened_at
}
);
}
/** Called in the revision transaction, before replacing the pinned bundle. */
export function archiveBetaRevision(id: number, revision: number, startedAt: string, notes: string | null): void {
db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)')
export function archiveBetaRevision(
id: number,
revision: number,
startedAt: string,
notes: string | null
): void {
db()
.prepare(
'INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)'
)
.run(id, revision, startedAt, notes);
db().prepare(`INSERT OR IGNORE INTO beta_revision_files
db()
.prepare(
`INSERT OR IGNORE INTO beta_revision_files
(beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content_hash)
SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content_hash
FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id
WHERE bs.beta_id = ?`).run(revision, id);
WHERE bs.beta_id = ?`
)
.run(revision, id);
}
export function betaRevisionHistory(id: number): BetaRevision[] {
const rows = db().prepare('SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC')
.all(id) as Array<{ revision: number; started_at: string; notes: string | null; results_json: string | null }>;
const files = db().prepare(`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`).all(id) as Array<BetaRevision['files'][number] & { revision: number }>;
return rows.map(({ results_json, ...row }) => ({ ...row,
const rows = db()
.prepare(
'SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC'
)
.all(id) as Array<{
revision: number;
started_at: string;
notes: string | null;
results_json: string | null;
}>;
const files = db()
.prepare(
`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`
)
.all(id) as Array<BetaRevision['files'][number] & { revision: number }>;
return rows.map(({ results_json, ...row }) => ({
...row,
results: results_json ? JSON.parse(results_json) : null,
files: files.filter(file => file.revision === row.revision).map(({ revision, ...file }) => file)
files: files
.filter((file) => file.revision === row.revision)
.map(({ revision, ...file }) => file)
}));
}
+103 -40
View File
@@ -7,7 +7,7 @@ import { observationReader } from './status.js';
import type { ScriptRow } from './scripts.js';
const games = ['t5', 't6'] as const;
type Game = typeof games[number];
type Game = (typeof games)[number];
type Counts = Partial<Record<Game, number>>;
export function parseBetaInstanceCounts(value: unknown): Counts {
@@ -16,7 +16,12 @@ export function parseBetaInstanceCounts(value: unknown): Counts {
}
const counts: Counts = {};
for (const [game, count] of Object.entries(value)) {
if (!games.includes(game as Game) || !Number.isSafeInteger(count) || count < 1 || count > 1000) {
if (
!games.includes(game as Game) ||
!Number.isSafeInteger(count) ||
count < 1 ||
count > 1000
) {
throw new Error('Instance counts must be integers from 1 to 1000 for t5 or t6');
}
counts[game as Game] = count;
@@ -25,79 +30,137 @@ export function parseBetaInstanceCounts(value: unknown): Counts {
}
function matches(script: ScriptRow, server: ServerRow): boolean {
return !script.scope_type || !script.scope_value
|| (script.scope_type === 'game' ? server.game : server.map) === script.scope_value;
return (
!script.scope_type ||
!script.scope_value ||
(script.scope_type === 'game' ? server.game : server.map) === script.scope_value
);
}
/** Choose at submission time using the same fresh observations shown in Fleet. */
export function selectBetaServers(scripts: ScriptRow[], instancesPerGame: unknown): string[] {
const counts = parseBetaInstanceCounts(instancesPerGame), observe = observationReader(), now = Date.now();
const hosts = new Map(listHosts().map(host => [host.id, host]));
const busy = new Set((db().prepare(`SELECT server FROM server_deployments WHERE server_deleted_at IS NULL
const counts = parseBetaInstanceCounts(instancesPerGame),
observe = observationReader(),
now = Date.now();
const hosts = new Map(listHosts().map((host) => [host.id, host]));
const busy = new Set(
(
db()
.prepare(
`SELECT server FROM server_deployments WHERE server_deleted_at IS NULL
AND (state IN ('queued','waiting','running','awaiting-health','interrupted')
OR (action IN ('destroy','rebuild') AND state = 'failed'
AND stage IN ('Stopping and removing container','Removing server files and registry')))`)
.all() as { server: string }[]).map(row => row.server));
const candidates = listServers().flatMap(server => {
if (!counts[server.game as Game] || server.desired_state === 'stopped' || busy.has(server.name)
|| !scripts.some(script => matches(script, server)) || protectedWork(server.name)) return [];
const host = server.host_id ? hosts.get(server.host_id) : null;
if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission)) return [];
const observation = observe(server);
if (observation.health !== 'online' || observation.last_players === null
|| !freshStatusTimestamp(observation.last_seen_at, now)) return [];
// An occupied RCON observation must not rank as empty just because a
// competing telemetry report claims zero players.
const rconPlayers = freshStatusTimestamp(server.last_seen_at, now)
&& (!observation.telemetry_after || Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after))
? server.last_players ?? 0 : 0;
return [{ server, players: Math.max(observation.last_players, rconPlayers) }];
}).sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name));
AND stage IN ('Stopping and removing container','Removing server files and registry')))`
)
.all() as { server: string }[]
).map((row) => row.server)
);
const candidates = listServers()
.flatMap((server) => {
if (
!counts[server.game as Game] ||
server.desired_state === 'stopped' ||
busy.has(server.name) ||
!scripts.some((script) => matches(script, server)) ||
protectedWork(server.name)
)
return [];
const host = server.host_id ? hosts.get(server.host_id) : null;
if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission))
return [];
const observation = observe(server);
if (
observation.health !== 'online' ||
observation.last_players === null ||
!freshStatusTimestamp(observation.last_seen_at, now)
)
return [];
// An occupied RCON observation must not rank as empty just because a
// competing telemetry report claims zero players.
const rconPlayers =
freshStatusTimestamp(server.last_seen_at, now) &&
(!observation.telemetry_after ||
Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after))
? (server.last_players ?? 0)
: 0;
return [{ server, players: Math.max(observation.last_players, rconPlayers) }];
})
.sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name));
for (const game of games) {
const available = candidates.filter(candidate => candidate.server.game === game).length;
const available = candidates.filter((candidate) => candidate.server.game === game).length;
if ((counts[game] ?? 0) > available) {
throw new Error(`${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.`);
throw new Error(
`${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.`
);
}
}
for (const script of scripts) {
if (!candidates.some(candidate => matches(script, candidate.server))) {
throw new Error(`${script.name}: no available server in the requested games matches this file's scope.`);
if (!candidates.some((candidate) => matches(script, candidate.server))) {
throw new Error(
`${script.name}: no available server in the requested games matches this file's scope.`
);
}
}
// Map scopes need a representative even when the least populated servers
// all run the same map. Reserve enough remaining slots for every map.
const requiredMaps = new Set(scripts.filter(script => script.scope_type === 'map').map(script => script.scope_value!));
const requiredMaps = new Set(
scripts.filter((script) => script.scope_type === 'map').map((script) => script.scope_value!)
);
function canComplete(remaining: typeof candidates, slots: Counts, maps: Set<string>): boolean {
const options = new Map<string, Set<Game>>();
for (const game of games) {
if (remaining.filter(candidate => candidate.server.game === game).length < (slots[game] ?? 0)) return false;
if (
remaining.filter((candidate) => candidate.server.game === game).length <
(slots[game] ?? 0)
)
return false;
}
for (const map of maps) {
const choices = new Set(remaining.filter(candidate => candidate.server.map === map && (slots[candidate.server.game as Game] ?? 0) > 0)
.map(candidate => candidate.server.game as Game));
const choices = new Set(
remaining
.filter(
(candidate) =>
candidate.server.map === map &&
(slots[candidate.server.game as Game] ?? 0) > 0
)
.map((candidate) => candidate.server.game as Game)
);
if (!choices.size) return false;
options.set(map, choices);
}
// There are two supported games: single-game maps reserve that game's
// slots, and maps available in either game use any remaining slot.
return games.every(game => [...options.values()].filter(choices => choices.size === 1 && choices.has(game)).length <= (slots[game] ?? 0))
&& maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0);
return (
games.every(
(game) =>
[...options.values()].filter(
(choices) => choices.size === 1 && choices.has(game)
).length <= (slots[game] ?? 0)
) && maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0)
);
}
if (!canComplete(candidates, counts, requiredMaps)) {
throw new Error('Increase the instance counts to cover every selected file\'s map scope, or choose servers in advanced options.');
throw new Error(
"Increase the instance counts to cover every selected file's map scope, or choose servers in advanced options."
);
}
const selected: string[] = [], slots = { ...counts };
const selected: string[] = [],
slots = { ...counts };
let maps = requiredMaps;
for (let index = 0; index < candidates.length; index++) {
const { server } = candidates[index], game = server.game as Game;
const { server } = candidates[index],
game = server.game as Game;
if (!slots[game]) continue;
const nextSlots = { ...slots, [game]: slots[game]! - 1 }, nextMaps = new Set(maps);
const nextSlots = { ...slots, [game]: slots[game]! - 1 },
nextMaps = new Set(maps);
if (server.map) nextMaps.delete(server.map);
if (!canComplete(candidates.slice(index + 1), nextSlots, nextMaps)) continue;
selected.push(server.name); slots[game] = nextSlots[game]; maps = nextMaps;
if (games.every(game => !slots[game])) break;
selected.push(server.name);
slots[game] = nextSlots[game];
maps = nextMaps;
if (games.every((game) => !slots[game])) break;
}
return selected;
}
+145 -36
View File
@@ -4,52 +4,120 @@ import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
import type { GameTelemetry } from '../../telemetry.js';
import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
import {
applySample,
applyGameEvent,
emptyMetrics,
evaluateRequirements,
type TestMetrics
} from './beta-evaluation.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { currentBetaRevision } from './beta-revisions.js';
import { invalidateBeta } from '../pipeline/bus.js';
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
export interface TestRun {
id: string;
beta_id: number;
server: string;
phase: number;
created_at: string;
observing_at: string | null;
metrics: string;
}
export interface StoredPlan {
beta_id: number;
plan: string;
outcome: string;
reason: string | null;
}
const nowIso = () => new Date().toISOString();
const activeParticipant = (id: number, server: string) => !!db().prepare(`SELECT 1 FROM beta_servers
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`).get(id, server);
export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined;
export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined;
export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); invalidateBeta(id); }
const activeParticipant = (id: number, server: string) =>
!!db()
.prepare(
`SELECT 1 FROM beta_servers
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
)
.get(id, server);
export const storedPlan = (id: number) =>
db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as
StoredPlan | undefined;
export const currentRun = (id: number, server: string) =>
db()
.prepare(
'SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL'
)
.get(id, server) as TestRun | undefined;
export function activity(id: number, message: string) {
db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message);
invalidateBeta(id);
}
export function setOutcome(id: number, outcome: string, reason: string | null): void {
const previous = storedPlan(id);
if (!previous || (previous.outcome === outcome && previous.reason === reason)) return;
db().prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?').run(outcome, reason, id);
db()
.prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?')
.run(outcome, reason, id);
activity(id, `Test plan ${outcome}${reason ? `: ${reason}` : ''}`);
}
export function readMetrics(run: TestRun): TestMetrics { return { ...emptyMetrics(), ...JSON.parse(run.metrics) }; }
export function saveMetrics(run: TestRun, metrics: TestMetrics): void { db().prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?').run(JSON.stringify(metrics), run.id); invalidateBeta(run.beta_id); }
export function readMetrics(run: TestRun): TestMetrics {
return { ...emptyMetrics(), ...JSON.parse(run.metrics) };
}
export function saveMetrics(run: TestRun, metrics: TestMetrics): void {
db()
.prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?')
.run(JSON.stringify(metrics), run.id);
invalidateBeta(run.beta_id);
}
export function evidence(run: TestRun, eventId: string, kind: string, payload: unknown): boolean {
return db().prepare('INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)')
.run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0;
return (
db()
.prepare(
'INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)'
)
.run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0
);
}
/** Internal progress record; never sent to or required from the game script. */
export async function prepareRun(betaId: number, server: string, phase = 0): Promise<TestRun> {
if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta');
if (!activeParticipant(betaId, server))
throw new Error('Original test server was deleted; start a new beta');
db().transaction(() => {
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server);
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), currentBetaRevision(betaId).revision);
db()
.prepare(
'UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL'
)
.run(nowIso(), betaId, server);
db()
.prepare(
'INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)'
)
.run(
randomUUID(),
betaId,
server,
phase,
nowIso(),
JSON.stringify(emptyMetrics()),
currentBetaRevision(betaId).revision
);
})();
return currentRun(betaId, server)!;
}
/** Route ordinary server events to every deployed, active beta automatically. */
export function recordGameEvent(event: GameTelemetry): void {
const runs = db().prepare(`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
const runs = db()
.prepare(
`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
JOIN beta_test_plans p ON p.beta_id = b.id
JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server
WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote')
AND s.server_deleted_at IS NULL
AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready')
AND s.restarted_at IS NOT NULL`).all(event.server) as TestRun[];
AND s.restarted_at IS NOT NULL`
)
.all(event.server) as TestRun[];
for (const run of runs) {
const plan = parseBetaPlan(JSON.parse(storedPlan(run.beta_id)!.plan));
const metrics = readMetrics(run);
@@ -64,17 +132,27 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
const m = readMetrics(run);
const row = getServer(run.server);
const server = row ? serverObservation(row) : null;
const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
const timestamp =
server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
const sample = Date.parse(timestamp ?? '');
const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
&& Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS;
const freshSample =
Number.isFinite(sample) &&
sample >= Date.parse(run.observing_at ?? run.created_at) &&
Date.now() - sample >= 0 &&
Date.now() - sample <= ONLINE_MAX_MS;
const healthy = freshSample && server?.health === 'online';
const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
const before = m.wasHealthy;
const lastSample = m.lastSample;
applySample(plan, m, at, healthy);
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, source: server?.status_source });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
if (freshSample && sample !== lastSample)
evidence(run, `sample:${sample}`, 'status-sample', {
healthy,
checkedAt: timestamp,
source: server?.status_source
});
if (before !== m.wasHealthy)
evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
saveMetrics(run, m);
}
@@ -82,22 +160,53 @@ export function testView(betaId: number): BetaTestView | null {
const stored = storedPlan(betaId);
if (!stored) return null;
const plan = parseBetaPlan(JSON.parse(stored.plan));
const runs = db().prepare(`SELECT r.*, s.server_deleted_at FROM beta_test_runs r
const runs = db()
.prepare(
`SELECT r.*, s.server_deleted_at FROM beta_test_runs r
JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`).all(betaId) as Array<TestRun & { server_deleted_at: string | null }>;
return { plan, outcome: stored.outcome, reason: stored.reason,
servers: runs.map(run => {
WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`
)
.all(betaId) as Array<TestRun & { server_deleted_at: string | null }>;
return {
plan,
outcome: stored.outcome,
reason: stored.reason,
servers: runs.map((run) => {
const metrics = readMetrics(run);
const progress = evaluateRequirements(plan, metrics, run.phase);
const latest = db().prepare("SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1").get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
const latest = db()
.prepare(
"SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1"
)
.get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
const event = latest ? JSON.parse(latest.payload) : null;
return { server: run.server, server_deleted_at: run.server_deleted_at, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at,
observation: { lastGameplayAt: latest?.received_at ?? null, lastEvent: latest?.kind ?? null,
players: metrics.match?.players ?? null, round: event?.round ?? null,
matchStarted: metrics.match?.started !== undefined, gameEnded: metrics.match?.ended ?? false,
healthy: metrics.wasHealthy ?? null, healthCheckedAt: metrics.lastSample ? new Date(metrics.lastSample).toISOString() : null } };
return {
server: run.server,
server_deleted_at: run.server_deleted_at,
phase: run.phase,
passed: progress.every((r) => r.passed),
progress,
observingAt: run.observing_at,
observation: {
lastGameplayAt: latest?.received_at ?? null,
lastEvent: latest?.kind ?? null,
players: metrics.match?.players ?? null,
round: event?.round ?? null,
matchStarted: metrics.match?.started !== undefined,
gameEnded: metrics.match?.ended ?? false,
healthy: metrics.wasHealthy ?? null,
healthCheckedAt: metrics.lastSample
? new Date(metrics.lastSample).toISOString()
: null
}
};
}),
evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
evidence: db()
.prepare(
`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`).all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence'] };
WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`
)
.all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence']
};
}
File diff suppressed because it is too large Load Diff
+184 -49
View File
@@ -1,32 +1,61 @@
import { createHash } from 'node:crypto';
import { db } from '../db/index.js';
import { enrichGameChatTimeline, type ChatMatch, type ChatRoundReport, type ContextualChatEntry } from '../../game-chat.js';
import {
enrichGameChatTimeline,
type ChatMatch,
type ChatRoundReport,
type ContextualChatEntry
} from '../../game-chat.js';
import { parseGameEvent } from '../../telemetry.js';
import { telemetryRetention, telemetryCutoff } from './telemetry-requests.js';
import { getServer, listServers, type ServerRow } from './servers.js';
import { gameLogKey, resolveLogPath, readLogChunk, type LogCursor, type TailResult } from './logs.js';
import {
gameLogKey,
resolveLogPath,
readLogChunk,
type LogCursor,
type TailResult
} from './logs.js';
import { listHosts } from '../hosts/index.js';
import { observationReader } from './status.js';
/** Historical context is scoped to this registration, including round-one matches. */
export function serverChatTimeline(server: ServerRow, tail: TailResult): ContextualChatEntry[] {
const unscoped = enrichGameChatTimeline(tail.lines, { mtime: tail.mtime, matches: [], reports: [] });
const times = unscoped.flatMap(entry => entry.context.realTime ? [entry.context.realTime] : []);
const unscoped = enrichGameChatTimeline(tail.lines, {
mtime: tail.mtime,
matches: [],
reports: []
});
const times = unscoped.flatMap((entry) =>
entry.context.realTime ? [entry.context.realTime] : []
);
if (!times.length) return unscoped;
const since = times.reduce((a, b) => a < b ? a : b), until = times.reduce((a, b) => a > b ? a : b);
const since = times.reduce((a, b) => (a < b ? a : b)),
until = times.reduce((a, b) => (a > b ? a : b));
const handle = db();
const matches = handle.prepare(`SELECT id, started_at AS startedAt, ended_at AS endedAt,
const matches = handle
.prepare(
`SELECT id, started_at AS startedAt, ended_at AS endedAt,
last_received_at AS lastReceivedAt, status, start_observed AS startObserved
FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL
AND started_at <= ? AND (COALESCE(ended_at,last_received_at) >= ? OR status='active')
ORDER BY id DESC LIMIT 2000`).all(server.name, until, since) as Array<Omit<ChatMatch, 'startObserved'> & { startObserved: number }>;
ORDER BY id DESC LIMIT 2000`
)
.all(server.name, until, since) as Array<
Omit<ChatMatch, 'startObserved'> & { startObserved: number }
>;
// Diagnostic history survives deletion. Its receipt time must belong to the
// current registration so a reused server name cannot inherit old rounds.
const cutoff = telemetryCutoff();
const roundSince = new Date(Date.parse(since) - 120_000).toISOString();
const matchSince = matches.reduce((earliest, match) => match.startedAt < earliest ? match.startedAt : earliest, since);
const reports = handle.prepare(`SELECT received_at AS receivedAt, event, round,
const matchSince = matches.reduce(
(earliest, match) => (match.startedAt < earliest ? match.startedAt : earliest),
since
);
const reports = handle
.prepare(
`SELECT received_at AS receivedAt, event, round,
player_count AS playerCount FROM telemetry_requests
WHERE server = ? AND status = 200 AND received_at >= ?
AND julianday(received_at) >= julianday(?)
@@ -34,36 +63,69 @@ export function serverChatTimeline(server: ServerRow, tail: TailResult): Context
AND received_at <= ?
AND event IN ('game_start', 'round_change', 'heartbeat', 'game_end')
AND player_count >= 0 AND (event = 'game_start' OR round >= 0)
ORDER BY id DESC LIMIT ?`).all(server.name, cutoff, server.created_at, roundSince, matchSince,
ORDER BY id DESC LIMIT ?`
)
.all(
server.name,
cutoff,
server.created_at,
roundSince,
matchSince,
new Date(Date.parse(until) + 120_000).toISOString(),
(telemetryRetention().maxPerServer || 10000)) as ChatRoundReport[];
telemetryRetention().maxPerServer || 10000
) as ChatRoundReport[];
// Saving gameplay does not depend on diagnostic logging succeeding. Keep the
// surviving accepted snapshot useful when its request log was not recorded.
const latest = handle.prepare(`SELECT received_at AS receivedAt, payload FROM server_telemetry
WHERE server = ? AND received_at >= ? AND julianday(received_at) >= julianday(?)`)
.get(server.name, cutoff, server.created_at) as { receivedAt: string; payload: string | null } | undefined;
const latest = handle
.prepare(
`SELECT received_at AS receivedAt, payload FROM server_telemetry
WHERE server = ? AND received_at >= ? AND julianday(received_at) >= julianday(?)`
)
.get(server.name, cutoff, server.created_at) as
{ receivedAt: string; payload: string | null } | undefined;
if (latest?.payload) {
try {
const event = parseGameEvent(JSON.parse(latest.payload));
const round = event.event === 'game_start' ? null : event.round;
if (event.server === server.name && !reports.some(report => report.receivedAt === latest.receivedAt
&& report.event === event.event && report.round === round && report.playerCount === event.players.length)) {
reports.push({ receivedAt: latest.receivedAt, event: event.event, round, playerCount: event.players.length });
if (
event.server === server.name &&
!reports.some(
(report) =>
report.receivedAt === latest.receivedAt &&
report.event === event.event &&
report.round === round &&
report.playerCount === event.players.length
)
) {
reports.push({
receivedAt: latest.receivedAt,
event: event.event,
round,
playerCount: event.players.length
});
}
} catch { /* Legacy malformed snapshots cannot establish chat context. */ }
} catch {
/* Legacy malformed snapshots cannot establish chat context. */
}
}
return enrichGameChatTimeline(tail.lines, {
mtime: tail.mtime,
matches: matches.map(match => ({ ...match, startObserved: Boolean(match.startObserved) })),
matches: matches.map((match) => ({
...match,
startObserved: Boolean(match.startObserved)
})),
reports
});
}
export function matchChatTimeline(matchId: number): ContextualChatEntry[] {
return (db().prepare('SELECT entry FROM match_chat WHERE match_id = ? ORDER BY rowid')
.all(matchId) as { entry: string }[]).map(row => JSON.parse(row.entry));
return (
db()
.prepare('SELECT entry FROM match_chat WHERE match_id = ? ORDER BY rowid')
.all(matchId) as { entry: string }[]
).map((row) => JSON.parse(row.entry));
}
/** Save before filtering the live view so completed matches retain their messages. */
@@ -86,15 +148,24 @@ function archiveChat(entries: ContextualChatEntry[]): void {
})();
}
export function currentMatchChat(server: ServerRow, tail: TailResult | null): ContextualChatEntry[] {
export function currentMatchChat(
server: ServerRow,
tail: TailResult | null
): ContextualChatEntry[] {
const entries = tail ? serverChatTimeline(server, tail) : [];
archiveChat(entries);
return currentChatEntries(server, entries);
}
function currentChatEntries(server: ServerRow, entries: ContextualChatEntry[]): ContextualChatEntry[] {
const match = db().prepare(`SELECT id, started_at AS startedAt FROM leaderboard_matches
WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1`)
function currentChatEntries(
server: ServerRow,
entries: ContextualChatEntry[]
): ContextualChatEntry[] {
const match = db()
.prepare(
`SELECT id, started_at AS startedAt FROM leaderboard_matches
WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1`
)
.get(server.name) as { id: number; startedAt: string } | undefined;
let startIndex = entries.length - 1;
while (startIndex >= 0) {
@@ -105,27 +176,44 @@ function currentChatEntries(server: ServerRow, entries: ContextualChatEntry[]):
const current = entries.slice(Math.max(0, startIndex));
if (!match) {
// Servers without gameplay telemetry still have log-defined sessions.
return current.filter(entry => entry.context.realTime !== null
&& Date.parse(entry.context.realTime) >= Date.parse(server.created_at));
return current.filter(
(entry) =>
entry.context.realTime !== null &&
Date.parse(entry.context.realTime) >= Date.parse(server.created_at)
);
}
const start = entries[startIndex];
// A log restart can arrive before game_start telemetry. Clear immediately
// instead of falling back to messages from the previous recorded match.
if (start && !current.some(entry => entry.matchId === match.id)
&& (!start.context.realTime || Date.parse(start.context.realTime) > Date.parse(match.startedAt) + 1000)) return [];
if (
start &&
!current.some((entry) => entry.matchId === match.id) &&
(!start.context.realTime ||
Date.parse(start.context.realTime) > Date.parse(match.startedAt) + 1000)
)
return [];
return matchChatTimeline(match.id);
}
const pending = new Map<string, Promise<{ entries: ContextualChatEntry[]; missing: boolean }>>();
interface ChatReadState {
key: string; cursor?: LogCursor; lines: string[]; partial: Buffer; skipLine: boolean;
signature?: string; entries: ContextualChatEntry[]; contextVersion?: string; wasOccupied: boolean;
key: string;
cursor?: LogCursor;
lines: string[];
partial: Buffer;
skipLine: boolean;
signature?: string;
entries: ContextualChatEntry[];
contextVersion?: string;
wasOccupied: boolean;
}
const readers = new Map<string, ChatReadState>();
const failures = new Map<string, number>();
/** Shared by the live API and the worker; collection does not require a viewer. */
export function collectServerChat(server: ServerRow): Promise<{ entries: ContextualChatEntry[]; missing: boolean }> {
export function collectServerChat(
server: ServerRow
): Promise<{ entries: ContextualChatEntry[]; missing: boolean }> {
const key = `${server.name}:${server.created_at}:${server.native_instance_id}:${server.instance_generation}`;
const existing = pending.get(key);
if (existing) return existing;
@@ -135,12 +223,26 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
let saved = readers.get(server.name);
if (saved?.key !== key) saved = undefined;
// Publish the new cursor only after its messages have been processed.
const state: ChatReadState = saved ? { ...saved } : { key, lines: [], partial: Buffer.alloc(0), skipLine: false, entries: [], wasOccupied: false };
const state: ChatReadState = saved
? { ...saved }
: {
key,
lines: [],
partial: Buffer.alloc(0),
skipLine: false,
entries: [],
wasOccupied: false
};
const tail = await readLogChunk(path, state.cursor);
const current = getServer(server.name);
// A remote read may finish after destruction or replacement.
if (!current || current.destroyed_at || current.created_at !== server.created_at
|| current.native_instance_id !== server.native_instance_id || current.instance_generation !== server.instance_generation) {
if (
!current ||
current.destroyed_at ||
current.created_at !== server.created_at ||
current.native_instance_id !== server.native_instance_id ||
current.instance_generation !== server.instance_generation
) {
return { entries: [], missing: true };
}
if (!tail) {
@@ -149,22 +251,40 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
}
let changed = false;
if ('cursor' in tail) {
if (tail.reset) { state.lines = []; state.partial = Buffer.alloc(0); state.skipLine = tail.truncated; state.entries = []; }
if (tail.reset) {
state.lines = [];
state.partial = Buffer.alloc(0);
state.skipLine = tail.truncated;
state.entries = [];
}
state.cursor = tail.cursor;
const bytes = Buffer.concat([state.partial, Buffer.from(tail.contentBase64, 'base64')]);
const end = bytes.lastIndexOf(10) + 1;
if (end) {
const lines = bytes.subarray(0, end).toString('utf8').split('\n').slice(0, -1).map(line => line.replace(/\r$/, ''));
const lines = bytes
.subarray(0, end)
.toString('utf8')
.split('\n')
.slice(0, -1)
.map((line) => line.replace(/\r$/, ''));
if (state.skipLine) lines.shift();
state.skipLine = false;
state.lines = [...state.lines, ...lines].slice(-2000);
let length = 0, start = state.lines.length;
while (start > 0 && (length += Buffer.byteLength(state.lines[start - 1]) + 1) <= 256 * 1024) start--;
let length = 0,
start = state.lines.length;
while (
start > 0 &&
(length += Buffer.byteLength(state.lines[start - 1]) + 1) <= 256 * 1024
)
start--;
state.lines = state.lines.slice(start);
changed = true;
}
state.partial = Buffer.from(bytes.subarray(end));
if (state.partial.length > 256 * 1024) { state.partial = Buffer.alloc(0); state.skipLine = true; }
if (state.partial.length > 256 * 1024) {
state.partial = Buffer.alloc(0);
state.skipLine = true;
}
// File mtime anchors the final line only after the backlog has been read.
if (tail.cursor.offset < tail.size || state.partial.length) {
state.signature = undefined;
@@ -177,9 +297,21 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
if (changed) state.lines = tail.lines;
}
const signature = `${tail.size}:${tail.mtime}`;
const needsContext = state.entries.some(entry => entry.type === 'message' && entry.matchId === undefined);
const contextVersion = needsContext ? (db().prepare('SELECT received_at FROM server_telemetry WHERE server=?').get(server.name) as { received_at: string } | undefined)?.received_at : undefined;
if (changed || state.signature === undefined || (needsContext && contextVersion !== state.contextVersion)) {
const needsContext = state.entries.some(
(entry) => entry.type === 'message' && entry.matchId === undefined
);
const contextVersion = needsContext
? (
db()
.prepare('SELECT received_at FROM server_telemetry WHERE server=?')
.get(server.name) as { received_at: string } | undefined
)?.received_at
: undefined;
if (
changed ||
state.signature === undefined ||
(needsContext && contextVersion !== state.contextVersion)
) {
state.entries = serverChatTimeline(current, { ...tail, lines: state.lines });
archiveChat(state.entries);
state.contextVersion = contextVersion;
@@ -193,15 +325,16 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
}
export async function collectFleetChat(): Promise<void> {
const hosts = new Map(listHosts().map(host => [host.id, host]));
const hosts = new Map(listHosts().map((host) => [host.id, host]));
const observe = observationReader();
const live = listServers().filter(server => !server.destroyed_at);
const names = new Set(live.map(server => server.name));
const live = listServers().filter((server) => !server.destroyed_at);
const names = new Set(live.map((server) => server.name));
for (const name of readers.keys()) if (!names.has(name)) readers.delete(name);
for (const name of failures.keys()) if (!names.has(name)) failures.delete(name);
const servers = live.filter(server => {
const servers = live.filter((server) => {
if (server.host_id && !hosts.get(server.host_id)?.connected) return false;
if (server.container_state && !['running', 'unknown'].includes(server.container_state)) return false;
if (server.container_state && !['running', 'unknown'].includes(server.container_state))
return false;
const players = observe(server).last_players ?? server.last_players;
// One final read after players leave retains messages written at match end.
return players !== 0 || readers.get(server.name)?.wasOccupied;
@@ -212,7 +345,9 @@ export async function collectFleetChat(): Promise<void> {
try {
await collectServerChat(server);
const state = readers.get(server.name);
if (state) state.wasOccupied = (observe(server).last_players ?? server.last_players ?? 0) > 0;
if (state)
state.wasOccupied =
(observe(server).last_players ?? server.last_players ?? 0) > 0;
failures.delete(server.name);
} catch (error) {
if (Date.now() - (failures.get(server.name) ?? -Infinity) >= 5 * 60_000) {
+420 -129
View File
@@ -1,7 +1,12 @@
import { scalingOwnershipBlocker } from '../autoscaling/ownership.js';
import { fleetSettings } from '../settings.js';
import { randomUUID } from 'node:crypto';
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
import type {
CollectionDetail,
CollectionMode,
CollectionPreview,
CollectionSummary
} from '$lib/deployment-plan.js';
import { db } from '../db/index.js';
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
import { creationQueue, destroyServer } from '../provisioning/index.js';
@@ -18,97 +23,203 @@ import { canCancelCreation, cancelCreation } from '../provisioning/cancel-creati
import { canCancelRemoval, cancelRemoval } from '../provisioning/history.js';
function coverage(entries: BlueprintEntry[], includeHealth = true) {
type Server = { name: string; game: string | null; map_slug: string | null; map: string | null };
const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[];
const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
type Server = {
name: string;
game: string | null;
map_slug: string | null;
map: string | null;
};
const registered = db()
.prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL')
.all() as Server[];
const queued = db()
.prepare(
`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[];
const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server));
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`
)
.all() as Server[];
const removing = new Set(
(
db()
.prepare(
"SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')"
)
.all() as { server: string }[]
).map((row) => row.server)
);
// A provisioning server can appear in both tables; count its identity once.
const servers = [...new Map([...queued, ...registered].map(server => [server.name, server])).values()];
return entries.map(entry => {
const existingServers = servers.filter(server => {
if (removing.has(server.name)) return false;
const parsed = parseServerName(server.name);
const game = server.game ?? parsed?.game;
if (game !== entry.recipe.game) return false;
if (server.map_slug) return server.map_slug === entry.recipe.mapSlug;
const namedMap = parsed?.mapCanonical ? zombieMapBySlug(game, parsed.mapSlug) : undefined;
if (namedMap && (!server.map || server.map === namedMap.canonical)) return namedMap.slug === entry.recipe.mapSlug;
// Legacy canonical maps are usable only when they identify one variant.
const matches = zombieMaps(game).filter(map => map.canonical === server.map);
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
}).map(server => server.name).sort();
const coverage = includeHealth ? existingServers.map(name => {
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
: server ? serverObservation(server).health : 'offline',
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
}) : [];
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
const servers = [
...new Map([...queued, ...registered].map((server) => [server.name, server])).values()
];
return entries.map((entry) => {
const existingServers = servers
.filter((server) => {
if (removing.has(server.name)) return false;
const parsed = parseServerName(server.name);
const game = server.game ?? parsed?.game;
if (game !== entry.recipe.game) return false;
if (server.map_slug) return server.map_slug === entry.recipe.mapSlug;
const namedMap = parsed?.mapCanonical
? zombieMapBySlug(game, parsed.mapSlug)
: undefined;
if (namedMap && (!server.map || server.map === namedMap.canonical))
return namedMap.slug === entry.recipe.mapSlug;
// Legacy canonical maps are usable only when they identify one variant.
const matches = zombieMaps(game).filter((map) => map.canonical === server.map);
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
})
.map((server) => server.name)
.sort();
const coverage = includeHealth
? existingServers.map((name) => {
const operation = db()
.prepare(
"SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1"
)
.get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return {
name,
health: operation
? operation.state === 'queued'
? 'queued'
: 'starting'
: server?.container_state &&
['stopped', 'exited', 'created', 'dead'].includes(
server.container_state
)
? 'stopped'
: server
? serverObservation(server).health
: 'offline',
href: operation
? `/provisioning/${encodeURIComponent(operation.id)}`
: `/servers/${encodeURIComponent(name)}`
};
})
: [];
return {
existingServers,
coverage,
existing: existingServers.length,
missing: Math.max(0, entry.count - existingServers.length)
};
});
}
function fleetSnapshot() {
return housePartyTargets().map(name => ({ name,
server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
return housePartyTargets().map((name) => ({
name,
server:
db()
.prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?')
.get(name) ?? null,
creation:
db()
.prepare(
"SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1"
)
.get(name) ?? null,
removal:
db()
.prepare(
"SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')"
)
.get(name) ?? null
}));
}
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, includeHealth = true) {
const counts = coverage(entries, includeHealth);
// Preserve deployed lobbies before queued ones, then the lowest lobby IDs.
const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers].sort((a, b) =>
Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true })
).slice(0, mode === 'exact' ? entries[index].count : undefined));
const retainedServers = counts.flatMap((entry, index) =>
[...entry.existingServers]
.sort(
(a, b) =>
Number(!!getServer(b)) - Number(!!getServer(a)) ||
a.localeCompare(b, 'en', { numeric: true })
)
.slice(0, mode === 'exact' ? entries[index].count : undefined)
);
const retained = new Set(retainedServers);
const matching = new Set(counts.flatMap(entry => entry.existingServers));
const removals = mode === 'exact' ? housePartyTargets().filter(name => !retained.has(name)).map(name => ({ name,
reason: matching.has(name) ? 'Exceeds the target lobby count' : 'Outside this collection or already awaiting removal'
})) : [];
const matching = new Set(counts.flatMap((entry) => entry.existingServers));
const removals =
mode === 'exact'
? housePartyTargets()
.filter((name) => !retained.has(name))
.map((name) => ({
name,
reason: matching.has(name)
? 'Exceeds the target lobby count'
: 'Outside this collection or already awaiting removal'
}))
: [];
return { counts, retainedServers, removals };
}
function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] {
const occupied = db().prepare(`SELECT name FROM servers UNION
function allocate(
entries: BlueprintEntry[],
counts = coverage(entries)
): CollectionPreview['servers'] {
const occupied = db()
.prepare(
`SELECT name FROM servers UNION
SELECT server AS name FROM native_instances WHERE retired_at IS NULL UNION
SELECT server_id AS name FROM server_key_leases UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`)
OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`
)
.all() as { name: string }[];
const lobbies = new Map<string, Set<number>>();
for (const { name } of occupied) {
const match = /^(.*)_(\d+)$/.exec(name);
if (!match) continue;
const taken = lobbies.get(match[1]) ?? new Set<number>();
taken.add(Number(match[2])); lobbies.set(match[1], taken);
taken.add(Number(match[2]));
lobbies.set(match[1], taken);
}
const availableKeys = { t5: availableServerKeyCount('t5'), t6: availableServerKeyCount('t6') };
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const taken = lobbies.get(base) ?? new Set<number>();
const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null;
const name = lobby === null ? null : `${base}_${lobby}`;
const native = entry.recipe.native;
const port = null;
if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0;
if (hasSavedKey) availableKeys[entry.recipe.game]--;
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey };
}));
return entries.flatMap((entry, entryIndex) =>
Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const taken = lobbies.get(base) ?? new Set<number>();
const lobby =
Array.from({ length: 20 }, (_, index) => index + 1).find(
(number) => !taken.has(number)
) ?? null;
const name = lobby === null ? null : `${base}_${lobby}`;
const native = entry.recipe.native;
const port = null;
if (lobby !== null) {
taken.add(lobby);
lobbies.set(base, taken);
}
const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0;
if (hasSavedKey) availableKeys[entry.recipe.game]--;
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey };
})
);
}
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
function blockersFor(
servers: CollectionPreview['servers'],
entries: BlueprintEntry[] = []
): string[] {
const blockers: string[] = [];
for (const server of servers) if (server.name) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); }
for (const server of servers)
if (server.name) {
const owned = scalingOwnershipBlocker(server.name);
if (owned) blockers.push(owned);
}
if (!servers.length) return blockers;
if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.');
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
if (servers.some((server) => !server.native && !entries[server.entryIndex]?.recipe.native))
blockers.push('Select and prepare a host before launching.');
if (servers.some((server) => !server.name))
blockers.push(
'There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.'
);
return blockers;
}
@@ -116,11 +227,16 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
const blockers: string[] = [];
const native = (name: string) => {
if (nativeInstance(name)) return true;
const creation = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined;
const creation = db()
.prepare(
`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`
)
.get(name) as { id: string } | undefined;
return !!creation && nativeDeployment(creation.id);
};
if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.');
if (removals.some((server) => !native(server.name)))
blockers.push('Every server to remove must have a native host instance.');
for (const server of removals) {
const owned = scalingOwnershipBlocker(server.name);
if (owned) blockers.push(owned);
@@ -130,50 +246,118 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
return blockers;
}
export function previewCollection(blueprintId: unknown, mode: unknown = 'fill', selection?: Record<string, unknown>): CollectionPreview {
if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.');
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
export function previewCollection(
blueprintId: unknown,
mode: unknown = 'fill',
selection?: Record<string, unknown>
): CollectionPreview {
if (typeof blueprintId !== 'string' || !blueprintId)
throw new Error('Choose a collection to launch.');
if (mode !== 'fill' && mode !== 'exact')
throw new Error('Choose fill or exact collection mode.');
return db().transaction((): CollectionPreview => {
const { entries, blueprintName } = blueprintEntries(blueprintId);
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
for (const entry of entries) {
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
entry.recipe.native = nativeLaunchSelection(
{ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId },
entry.recipe.game
);
entry.recipe.image = entry.recipe.native.image!;
}
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
const servers = allocate(entries, counts);
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
const selections = servers.flatMap(server => server.native ? [server.native] : []);
const selections = servers.flatMap((server) => (server.native ? [server.native] : []));
const availability = selections.length ? launchAvailability(selections) : {};
const id = randomUUID();
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null);
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ...availability, ready: blockers.length === 0,
entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) };
db()
.prepare(
'DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL'
)
.run(new Date().toISOString());
db()
.prepare(
'INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)'
)
.run(
id,
expiresAt,
blueprintName,
JSON.stringify(entries),
JSON.stringify(servers),
mode,
mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null
);
return {
id,
expiresAt,
blueprintName,
mode,
retainedServers,
removals,
launchConcurrency: fleetSettings().createConcurrency,
servers,
blockers,
...availability,
ready: blockers.length === 0,
entries: entries.map((entry, index) => ({
count: entry.count,
...counts[index],
recipe: summarizeRecipe(entry.recipe)
}))
};
})();
}
/** Save all reviewed identities and keys atomically; native placement belongs to provisioning. */
export function queueCollection(input: { previewId?: unknown; keys?: unknown }): { ok: true; collectionId: string | null } {
if (typeof input.previewId !== 'string' || !input.previewId) throw new Error('Review a collection preview before launching.');
if (input.keys !== undefined && (!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))) throw new Error('Provide keys indexed by the exact server ID.');
export function queueCollection(input: { previewId?: unknown; keys?: unknown }): {
ok: true;
collectionId: string | null;
} {
if (typeof input.previewId !== 'string' || !input.previewId)
throw new Error('Review a collection preview before launching.');
if (
input.keys !== undefined &&
(!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))
)
throw new Error('Provide keys indexed by the exact server ID.');
const keys = (input.keys ?? {}) as Record<string, unknown>;
return db().transaction(() => {
const preview = db().prepare('SELECT * FROM server_collection_previews WHERE id = ?').get(input.previewId) as {
id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null; mode: CollectionMode; fleet_snapshot: string | null;
} | undefined;
if (!preview) throw new Error('Collection preview expired or was not found. Refresh the preview.');
if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id };
if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.');
const preview = db()
.prepare('SELECT * FROM server_collection_previews WHERE id = ?')
.get(input.previewId) as
| {
id: string;
expires_at: string;
blueprint_name: string;
entries: string;
servers: string;
collection_id: string | null;
mode: CollectionMode;
fleet_snapshot: string | null;
}
| undefined;
if (!preview)
throw new Error('Collection preview expired or was not found. Refresh the preview.');
if (preview.collection_id)
return { ok: true as const, collectionId: preview.collection_id };
if (preview.expires_at <= new Date().toISOString())
throw new Error('Collection preview expired. Refresh it before launching.');
const entries: BlueprintEntry[] = JSON.parse(preview.entries);
// Older previews embedded plugin bytes. Queued jobs need only the
// reviewed metadata; provisioning captures the current files at dispatch.
for (const entry of entries) entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file);
for (const entry of entries)
entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file);
const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers);
if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) {
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
if (
preview.mode === 'exact' &&
preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())
) {
throw new Error(
'The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.'
);
}
const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false);
const servers = reviewed;
@@ -181,7 +365,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
for (const server of servers) {
const recipe = entries[server.entryIndex].recipe;
const selection = server.native ?? recipe.native;
if (!selection) throw new Error('Select a native host before launching. Refresh the preview.');
if (!selection)
throw new Error('Select a native host before launching. Refresh the preview.');
const identity = JSON.stringify([recipe.game, selection]);
if (validatedSelections.has(identity)) continue;
// Recheck saved configuration once per selection without allocating
@@ -191,50 +376,102 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
}
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
if (blockers.length) throw new Error(blockers.join(' '));
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) {
throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.');
if (
entries.some(
(_entry, index) =>
counts[index].missing !==
reviewed.filter((server) => server.entryIndex === index).length
)
) {
throw new Error(
'Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.'
);
}
const expected = new Set(servers.map(server => server.name));
const expected = new Set(servers.map((server) => server.name));
for (const [name, value] of Object.entries(keys)) {
if (!expected.has(name) || typeof value !== 'string') throw new Error('Keys must be strings for server IDs shown in this preview.');
if (!expected.has(name) || typeof value !== 'string')
throw new Error('Keys must be strings for server IDs shown in this preview.');
}
if (!servers.length && !removals.length) return { ok: true as const, collectionId: null };
const collectionId = randomUUID();
db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)')
.run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers));
db()
.prepare(
'INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)'
)
.run(
collectionId,
preview.blueprint_name,
new Date().toISOString(),
preview.mode,
JSON.stringify(retainedServers)
);
removals.forEach((server, position) => {
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
const queued = db()
.prepare(
"SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'"
)
.get(server.name) as { id: string } | undefined;
let id: string;
if (queued && !getServer(server.name)) {
// No container or files exist yet: cancel the reservation instead of booting just to remove it.
const result = destroyServer(server.name, 'blueprint-exact');
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
if (!result.deploymentId)
throw new Error('Could not queue removal for ' + server.name);
id = result.deploymentId;
db().prepare("UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?").run(new Date().toISOString(), queued.id);
recordDeploymentEvent(queued.id, 'Queued creation cancelled by exact collection; saved key retained.');
db()
.prepare(
"UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?"
)
.run(new Date().toISOString(), queued.id);
recordDeploymentEvent(
queued.id,
'Queued creation cancelled by exact collection; saved key retained.'
);
deploymentStage(id, 'Cancelled queued creation; no server was started');
finishDeployment(id, { ok: true });
} else {
const result = destroyServer(server.name, 'blueprint-exact');
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
if (!result.deploymentId)
throw new Error('Could not queue removal for ' + server.name);
id = result.deploymentId;
}
db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);
db()
.prepare(
'INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)'
)
.run(collectionId, id, position);
});
servers.forEach((server, index) => {
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.');
const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name });
const recipe = {
...entries[server.entryIndex].recipe,
...(server.native ? { native: server.native } : {})
};
if (!recipe.native)
throw new Error('Select a native host before launching. Refresh the preview.');
const result = queueNativeCreationIntent({
server: server.name!,
game: recipe.game,
mapSlug: recipe.mapSlug,
selection: recipe.native,
recipe,
plutoKey: keys[server.name!] as string | undefined,
createdBy: 'ui',
blueprintName: preview.blueprint_name
});
// Throw so the outer transaction also reverses all earlier keys and children.
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
if (!result.ok)
throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db()
.prepare(
'INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)'
)
.run(collectionId, result.deploymentId, index);
});
db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id);
db()
.prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?')
.run(collectionId, preview.id);
return { ok: true as const, collectionId };
})();
}
@@ -243,39 +480,60 @@ const membershipSql = `SELECT collection_id, deployment_id, position FROM server
UNION ALL SELECT collection_id, deployment_id, position FROM server_collection_removals`;
function collectionCancellationCandidates(id: string) {
const creations = (db().prepare(`SELECT d.id FROM server_collection_members m
const creations = (
db()
.prepare(
`SELECT d.id FROM server_collection_members m
JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.collection_id=?
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.creation_id=d.id)`)
.all(id) as { id: string }[]).filter(job => canCancelCreation(job.id));
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.creation_id=d.id)`
)
.all(id) as { id: string }[]
).filter((job) => canCancelCreation(job.id));
// Exact launches can reuse manual removals or creation cleanup jobs. Membership
// alone must never give a launch permission to withdraw those operations.
const removals = (db().prepare(`SELECT d.id FROM server_collection_removals m
const removals = (
db()
.prepare(
`SELECT d.id FROM server_collection_removals m
JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.collection_id=? AND d.triggered_by='blueprint-exact'
AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.operation_id IS NOT NULL)
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.cleanup_id=d.id)`)
.all(id) as { id: string }[]).filter(job => canCancelRemoval(job.id));
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.cleanup_id=d.id)`
)
.all(id) as { id: string }[]
).filter((job) => canCancelRemoval(job.id));
return { creations, removals };
}
/** Withdraw the remaining launch atomically; partial instances keep durable cleanup jobs. */
export function cancelCollection(id: string): boolean {
return db().transaction(() => {
const collection = db().prepare('SELECT cancelled_at FROM server_launch_collections WHERE id=?').get(id) as { cancelled_at: string | null } | undefined;
const collection = db()
.prepare('SELECT cancelled_at FROM server_launch_collections WHERE id=?')
.get(id) as { cancelled_at: string | null } | undefined;
if (!collection) return false;
if (collection.cancelled_at) return true;
const { creations, removals } = collectionCancellationCandidates(id);
if (!creations.length && !removals.length) return false;
db().prepare('UPDATE server_launch_collections SET cancelled_at=? WHERE id=?').run(new Date().toISOString(), id);
db()
.prepare('UPDATE server_launch_collections SET cancelled_at=? WHERE id=?')
.run(new Date().toISOString(), id);
for (const creation of creations) {
if (!cancelCreation(creation.id, 'collection-cancel')) throw new Error('Could not cancel a collection creation. Refresh the launch and try again.');
if (!cancelCreation(creation.id, 'collection-cancel'))
throw new Error(
'Could not cancel a collection creation. Refresh the launch and try again.'
);
}
for (const removal of removals) {
// A shared removal remains requested until every owning launch withdraws.
const shared = db().prepare(`SELECT 1 FROM server_collection_removals m
const shared = db()
.prepare(
`SELECT 1 FROM server_collection_removals m
JOIN server_launch_collections c ON c.id=m.collection_id
WHERE m.deployment_id=? AND c.id!=? AND c.cancelled_at IS NULL LIMIT 1`).get(removal.id, id);
WHERE m.deployment_id=? AND c.id!=? AND c.cancelled_at IS NULL LIMIT 1`
)
.get(removal.id, id);
if (!shared) cancelRemoval(removal.id);
}
return true;
@@ -299,26 +557,59 @@ const summarySql = `SELECT c.id, c.name, c.created_at, c.cancelled_at, c.mode, C
LEFT JOIN server_deployments cleanup ON cleanup.id = cc.cleanup_id`;
export function listCollections(): CollectionSummary[] {
return db().prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC').all() as CollectionSummary[];
return db()
.prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC')
.all() as CollectionSummary[];
}
export function getCollection(id: string): CollectionDetail | null {
const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined;
const collection = db()
.prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id')
.get(id) as CollectionSummary | undefined;
if (!collection) return null;
const servers = db().prepare(`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
const servers = db()
.prepare(
`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
(d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers'];
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`
)
.all(id) as CollectionDetail['servers'];
const queue = creationQueue();
const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1]));
const stored = db().prepare('SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string; retained_server_deletions: string };
const cancellations = db().prepare(`SELECT cc.creation_id, cc.cleanup_id AS cleanupId, d.state, d.stage, d.error
const stored = db()
.prepare(
'SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?'
)
.get(id) as { retained_servers: string; retained_server_deletions: string };
const cancellations = db()
.prepare(
`SELECT cc.creation_id, cc.cleanup_id AS cleanupId, d.state, d.stage, d.error
FROM server_collection_members m JOIN native_creation_cancellations cc ON cc.creation_id=m.deployment_id
JOIN server_deployments d ON d.id=cc.cleanup_id WHERE m.collection_id=?`).all(id) as
(NonNullable<CollectionDetail['servers'][number]['cancellation']> & { creation_id: string })[];
const cleanupByCreation = new Map(cancellations.map(({ creation_id, ...cleanup }) => [creation_id, cleanup]));
JOIN server_deployments d ON d.id=cc.cleanup_id WHERE m.collection_id=?`
)
.all(id) as (NonNullable<CollectionDetail['servers'][number]['cancellation']> & {
creation_id: string;
})[];
const cleanupByCreation = new Map(
cancellations.map(({ creation_id, ...cleanup }) => [creation_id, cleanup])
);
const candidates = collection.cancelled_at ? null : collectionCancellationCandidates(id);
return { collection, canCancel: !!candidates && (candidates.creations.length > 0 || candidates.removals.length > 0),
retainedServers: JSON.parse(stored.retained_servers), retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length },
servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null, cancellation: cleanupByCreation.get(server.id) ?? null })) };
return {
collection,
canCancel:
!!candidates && (candidates.creations.length > 0 || candidates.removals.length > 0),
retainedServers: JSON.parse(stored.retained_servers),
retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions),
queue: {
concurrency: fleetSettings().createConcurrency,
active: queue.active,
queued: queue.queued.length
},
servers: servers.map((server) => ({
...server,
queuePosition: positions.get(server.id) ?? null,
cancellation: cleanupByCreation.get(server.id) ?? null
}))
};
}
+72 -19
View File
@@ -1,62 +1,115 @@
import { composeServerCfg, parseCfg } from './config-compose.js';
import { selectedZombieMap, mapRotation } from './map-aliases.js';
import { configWarnings, type ConfigAssignment, type ConfigInspection, type ConfigSource } from '../../config-workbench.js';
import {
configWarnings,
type ConfigAssignment,
type ConfigInspection,
type ConfigSource
} from '../../config-workbench.js';
export interface ConfigLayer { source: ConfigSource; text: string; }
export interface ConfigLayer {
source: ConfigSource;
text: string;
}
/** Trace file composition, including assignments left in place by the composer.
* This is an intended-file preview, not a query of live runtime dvars. */
export function inspectConfig(baseline: ConfigLayer, overlays: ConfigLayer[], game: string | null, map: string | null, mapSlug?: string | null): ConfigInspection {
export function inspectConfig(
baseline: ConfigLayer,
overlays: ConfigLayer[],
game: string | null,
map: string | null,
mapSlug?: string | null
): ConfigInspection {
const sorted = [...overlays].sort((a, b) => a.source.id! - b.source.id!);
const parsed = parseCfg(baseline.text);
type TracedLine = { line: string; history: ConfigAssignment[] };
function assignment(line: string, source: ConfigSource, index: number): ConfigAssignment {
const match = /^\s*(?:(?:set|seta)\s+)(\S+)\s+(.*)$/i.exec(line)
?? /^\s*(sv_maprotation)\s+(.*)$/i.exec(line);
const match =
/^\s*(?:(?:set|seta)\s+)(\S+)\s+(.*)$/i.exec(line) ??
/^\s*(sv_maprotation)\s+(.*)$/i.exec(line);
return { value: match?.[2] ?? '', source, line: index + 1 };
}
const lines: TracedLine[] = parsed.order.map((entry, index) => ({
line: entry.line, history: [assignment(entry.line, baseline.source, index)]
line: entry.line,
history: [assignment(entry.line, baseline.source, index)]
}));
const replacements = new Map<string, TracedLine>();
const warnings = configWarnings(baseline.text, false).map(w => `${baseline.source.name}: ${w}`);
const warnings = configWarnings(baseline.text, false).map(
(w) => `${baseline.source.name}: ${w}`
);
for (const overlay of sorted) {
warnings.push(...configWarnings(overlay.text, true).map(w => `${overlay.source.name}: ${w}`));
warnings.push(
...configWarnings(overlay.text, true).map((w) => `${overlay.source.name}: ${w}`)
);
overlay.text.split(/\r?\n/).forEach((line, index) => {
const match = /^\s*set\s+(\S+)\s+/.exec(line);
if (!match) return;
const previous = replacements.get(match[1]);
replacements.set(match[1], { line, history: [...(previous?.history ?? []), assignment(line, overlay.source, index)] });
replacements.set(match[1], {
line,
history: [...(previous?.history ?? []), assignment(line, overlay.source, index)]
});
});
}
for (const [key, replacement] of replacements) {
const index = parsed.entries.get(key);
if (index === undefined) lines.push(replacement);
else lines[index] = { line: replacement.line, history: [...lines[index].history, ...replacement.history] };
else
lines[index] = {
line: replacement.line,
history: [...lines[index].history, ...replacement.history]
};
}
const sources = [baseline.source, ...sorted.map(o => o.source)];
const sources = [baseline.source, ...sorted.map((o) => o.source)];
const selected = selectedZombieMap(game, map, mapSlug);
if (selected) {
const source: ConfigSource = { id: null, name: 'Server map selection', version: null, kind: 'map' };
const source: ConfigSource = {
id: null,
name: 'Server map selection',
version: null,
kind: 'map'
};
sources.push(source);
const rotation = /^\s*(?:(?:set|seta)\s+)?sv_maprotation\s+/i;
const history = lines.filter(entry => rotation.test(entry.line)).flatMap(entry => entry.history);
for (let i = lines.length - 1; i >= 0; i--) if (rotation.test(lines[i].line)) lines.splice(i, 1);
const history = lines
.filter((entry) => rotation.test(entry.line))
.flatMap((entry) => entry.history);
for (let i = lines.length - 1; i >= 0; i--)
if (rotation.test(lines[i].line)) lines.splice(i, 1);
const line = `set sv_maprotation "${mapRotation(selected)}"`;
lines.push({ line, history: [...history, assignment(line, source, 0)] });
}
const byKey = new Map<string, { key: string; history: ConfigAssignment[] }>();
for (const entry of lines) {
const match = /^\s*(?:set|seta)\s+(\S+)\s+/i.exec(entry.line) ?? /^\s*(sv_maprotation)\s+/i.exec(entry.line);
const match =
/^\s*(?:set|seta)\s+(\S+)\s+/i.exec(entry.line) ??
/^\s*(sv_maprotation)\s+/i.exec(entry.line);
if (!match) continue;
const normalized = match[1].toLowerCase();
const existing = byKey.get(normalized);
byKey.set(normalized, { key: match[1], history: [...(existing?.history ?? []), ...entry.history] });
byKey.set(normalized, {
key: match[1],
history: [...(existing?.history ?? []), ...entry.history]
});
}
return {
text: composeServerCfg(baseline.text, sorted.map(o => ({ id: o.source.id!, text: o.text })), game, map, mapSlug),
sources, warnings,
settings: [...byKey.values()].map(({ key, history }) => ({ key, history, value: history.at(-1)!.value, source: history.at(-1)!.source }))
text: composeServerCfg(
baseline.text,
sorted.map((o) => ({ id: o.source.id!, text: o.text })),
game,
map,
mapSlug
),
sources,
warnings,
settings: [...byKey.values()]
.map(({ key, history }) => ({
key,
history,
value: history.at(-1)!.value,
source: history.at(-1)!.source
}))
.sort((a, b) => a.key.localeCompare(b.key))
};
}
+360 -114
View File
@@ -1,6 +1,12 @@
import { putFileBlob } from '../db/file-blobs.js';
import { getVersionContent } from './scripts.js';
import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js';
import {
readdir,
readFile,
stat,
nativeFileManifest,
type FileFingerprint
} from '../backends/files.js';
import { randomUUID } from 'node:crypto';
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
import { join, posix, relative, sep } from 'node:path';
@@ -27,12 +33,18 @@ async function scanBatch<T, R>(items: readonly T[], work: (item: T) => Promise<R
async function pump() {
while (!failed && cursor < items.length) {
const index = cursor++;
try { results[index] = await work(items[index]); }
catch (error) { failed = true; throw error; }
try {
results[index] = await work(items[index]);
} catch (error) {
failed = true;
throw error;
}
}
}
const workers = await Promise.allSettled(Array.from({ length: Math.min(4, items.length) }, pump));
const failure = workers.find(result => result.status === 'rejected');
const workers = await Promise.allSettled(
Array.from({ length: Math.min(4, items.length) }, pump)
);
const failure = workers.find((result) => result.status === 'rejected');
if (failure?.status === 'rejected') throw failure.reason;
return results;
}
@@ -73,15 +85,22 @@ function toPosix(p: string): string {
}
function hasActiveBeta(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
return !!db()
.prepare(
`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`
)
.get(server);
}
async function requireServerDirectory(server: string): Promise<void> {
try {
if (!(await stat(serverDir(server))).isDirectory()) throw new Error('path is not a directory');
if (!(await stat(serverDir(server))).isDirectory())
throw new Error('path is not a directory');
} catch (error) {
throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`);
throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`, {
cause: error
});
}
}
@@ -92,7 +111,10 @@ async function readExistingFile(path: string): Promise<Buffer | null> {
});
}
async function fingerprint(path: string, manifest: Map<string, FileFingerprint> | null): Promise<FileFingerprint | null> {
async function fingerprint(
path: string,
manifest: Map<string, FileFingerprint> | null
): Promise<FileFingerprint | null> {
if (manifest) return manifest.get(path) ?? null;
const content = await readExistingFile(path);
return content === null ? null : { size: content.byteLength, sha256: sha256Buffer(content) };
@@ -102,10 +124,11 @@ async function walkFiles(root: string): Promise<string[]> {
const out: string[] = [];
let directories = [root];
while (directories.length) {
const children = await scanBatch(directories, async dir => {
const children = await scanBatch(directories, async (dir) => {
let entries;
try { entries = await readdir(dir, { withFileTypes: true }); }
catch (error) {
try {
entries = await readdir(dir, { withFileTypes: true });
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [];
throw error;
}
@@ -131,51 +154,83 @@ async function walkFiles(root: string): Promise<string[]> {
* artifacts with the promoted versions.
*/
export async function scanServerDrift(server: string): Promise<DriftEntry[]> {
return trackActivity(`drift:${server}`, `Checking files on ${server}`, () => scanServerFiles(server));
return trackActivity(`drift:${server}`, `Checking files on ${server}`, () =>
scanServerFiles(server)
);
}
async function scanServerFiles(server: string): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv || srv.destroyed_at || hasActiveBeta(server)) return [];
// A missing mount is unavailable evidence, not an empty managed tree.
const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) => join(destOfKind(server, kind), script.target_rel_path));
if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id) expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server)));
if (srv.created_by && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE));
const manifest = await nativeFileManifest(serverDir(server), DRIFT_DESTS.map(kind => destOfKind(server, kind)), [...DRIFT_IGNORE_FILENAMES], expectedPaths);
const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) =>
join(destOfKind(server, kind), script.target_rel_path)
);
if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id)
expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server)));
if (
srv.created_by &&
srv.game_mode === 'zombie' &&
getServerMapConfig(srv.game, srv.map, srv.map_slug)
)
expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE));
const manifest = await nativeFileManifest(
serverDir(server),
DRIFT_DESTS.map((kind) => destOfKind(server, kind)),
[...DRIFT_IGNORE_FILENAMES],
expectedPaths
);
if (!manifest) await requireServerDirectory(server);
const drift = await scanManagedForServer(server, manifest);
// Previous inventory is an audit of installed bytes, not permission to
// retain artifacts whose scope was moved or removed from the library.
const managed = new Set(promotedArtifactMetadata(srv.game, srv.map)
.map(({ kind, script }) => `${kind}:${script.target_rel_path}`));
const managed = new Set(
promotedArtifactMetadata(srv.game, srv.map).map(
({ kind, script }) => `${kind}:${script.target_rel_path}`
)
);
const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined;
if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`);
// Without a promoted baseline, Plutainer owns the seeded config. Keep it
// until a baseline is promoted; startup arguments handle map selection.
const keepDefaultConfig = srv?.game
&& selectedZombieMap(srv.game, srv.map, srv.map_slug)
&& !baseline?.current_version_id;
const keepDefaultConfig =
srv?.game &&
selectedZombieMap(srv.game, srv.map, srv.map_slug) &&
!baseline?.current_version_id;
// Scan for untracked files only after resolving managed targets.
for (const kind of DRIFT_DESTS) {
const dest = destOfKind(server, kind);
const onDisk = manifest
? [...manifest.keys()].filter(path => path.startsWith(`${dest}${sep}`)).map(path => toPosix(relative(dest, path)))
.filter(path => !path.split('/').some(name => DRIFT_IGNORE_FILENAMES.has(name))).sort()
? [...manifest.keys()]
.filter((path) => path.startsWith(`${dest}${sep}`))
.map((path) => toPosix(relative(dest, path)))
.filter(
(path) => !path.split('/').some((name) => DRIFT_IGNORE_FILENAMES.has(name))
)
.sort()
: await walkFiles(dest);
if (onDisk.length === 0) continue;
const untracked = await scanBatch(onDisk, async (relPath): Promise<DriftEntry | null> => {
if (managed.has(`${kind}:${relPath}`)) return null;
if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) return null;
if (kind === 'config' && relPath === MAP_CONFIG_FILE && srv?.created_by
&& srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) return null;
if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server))
return null;
if (
kind === 'config' &&
relPath === MAP_CONFIG_FILE &&
srv?.created_by &&
srv.game_mode === 'zombie' &&
getServerMapConfig(srv.game, srv.map, srv.map_slug)
)
return null;
const abs = join(dest, relPath);
try {
const info = await fingerprint(abs, manifest);
if (!info) return null;
return {
server, relPath,
server,
relPath,
...info,
kind,
driftKind: 'untracked'
@@ -196,38 +251,63 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
// Every promoted artifact whose scope matches this server's game/map should
// have its target file present on disk. Anything absent is 'missing' drift.
// Configs are handled as a composed layer (baseline + overlays → one cfg).
async function scanManagedForServer(server: string, manifest: Map<string, FileFingerprint> | null): Promise<DriftEntry[]> {
async function scanManagedForServer(
server: string,
manifest: Map<string, FileFingerprint> | null
): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv) return [];
const out: DriftEntry[] = [];
// Provisioned servers load this generated file after their main cfg. It
// remains managed even when there is no promoted baseline yet.
const mapConfig = srv.created_by && srv.game_mode === 'zombie'
? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null;
const mapConfig =
srv.created_by && srv.game_mode === 'zombie'
? getServerMapConfig(srv.game, srv.map, srv.map_slug)
: null;
if (mapConfig) {
const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE);
const content = await fingerprint(abs, manifest);
if (content?.sha256 !== sha256Buffer(mapConfig)) {
out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing',
size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) });
out.push({
server,
relPath: MAP_CONFIG_FILE,
kind: 'config',
driftKind: content ? 'replace' : 'missing',
size: mapConfig.byteLength,
sha256: sha256Buffer(mapConfig)
});
}
}
// Inventory records the last managed write, not the current disk bytes.
// Check both so edits and outdated versions are detected on tracked files.
const managed = await scanBatch(promotedArtifactMetadata(srv.game, srv.map), async ({ kind, script, version }): Promise<DriftEntry | null> => {
const content = await fingerprint(join(destOfKind(server, kind), script.target_rel_path), manifest);
const inventory = getServerScript(server, script.target_rel_path);
if (content?.sha256 === version.sha256 && inventory?.sha256 === version.sha256
&& inventory.script_version_id === version.id) return null;
return {
server, relPath: script.target_rel_path, kind,
driftKind: content ? 'replace' : 'missing',
size: version.size, sha256: version.sha256,
scriptId: script.id, scriptVersionId: version.id
};
});
const managed = await scanBatch(
promotedArtifactMetadata(srv.game, srv.map),
async ({ kind, script, version }): Promise<DriftEntry | null> => {
const content = await fingerprint(
join(destOfKind(server, kind), script.target_rel_path),
manifest
);
const inventory = getServerScript(server, script.target_rel_path);
if (
content?.sha256 === version.sha256 &&
inventory?.sha256 === version.sha256 &&
inventory.script_version_id === version.id
)
return null;
return {
server,
relPath: script.target_rel_path,
kind,
driftKind: content ? 'replace' : 'missing',
size: version.size,
sha256: version.sha256,
scriptId: script.id,
scriptVersionId: version.id
};
}
);
out.push(...managed.filter((entry): entry is DriftEntry => entry !== null));
// Composed cfg: the effective dedicated_sp.cfg is baseline + all promoted
@@ -247,13 +327,24 @@ async function scanManagedForServer(server: string, manifest: Map<string, FileFi
// Older launches tracked the promoted bytes before Plutainer
// applied the instance password. Reconcile only that known
// before-image, and only after verifying the full effective file.
if (composed.sha256 !== composed.promotedSha256 && content?.sha256 === composed.sha256
&& inv?.sha256 === composed.promotedSha256 && !hasActiveBeta(server)) {
recordScript(server, targetRel, inv.script_version_id, composed.sha256,
composed.content.byteLength, 'runtime-config-reconciliation');
if (
composed.sha256 !== composed.promotedSha256 &&
content?.sha256 === composed.sha256 &&
inv?.sha256 === composed.promotedSha256 &&
!hasActiveBeta(server)
) {
recordScript(
server,
targetRel,
inv.script_version_id,
composed.sha256,
composed.content.byteLength,
'runtime-config-reconciliation'
);
inv = getServerScript(server, targetRel);
}
const onDiskMatches = content?.sha256 === composed.sha256 && inv?.sha256 === composed.sha256;
const onDiskMatches =
content?.sha256 === composed.sha256 && inv?.sha256 === composed.sha256;
if (!onDiskMatches) {
out.push({
server,
@@ -282,7 +373,8 @@ async function resolveManagedEntry(server: string, entry: DriftEntry): Promise<B
if (entry.kind === 'config') {
const srv = getServer(server);
if (!srv?.game) return null;
if (entry.relPath === MAP_CONFIG_FILE) return getServerMapConfig(srv.game, srv.map, srv.map_slug);
if (entry.relPath === MAP_CONFIG_FILE)
return getServerMapConfig(srv.game, srv.map, srv.map_slug);
const composed = composeExpectedCfg(server, srv.game, srv.map, srv.map_slug);
return composed?.content ?? null;
}
@@ -299,7 +391,9 @@ function composeExpectedCfg(
): { content: Buffer; sha256: string; promotedSha256: string } | null {
const promoted = getPromotedServerConfig(game, map, mapSlug);
if (!promoted) return null;
const content = Buffer.from(applyRuntimePassword(promoted.toString('utf8'), runtimePasswordForServer(server)));
const content = Buffer.from(
applyRuntimePassword(promoted.toString('utf8'), runtimePasswordForServer(server))
);
return { content, sha256: sha256Buffer(content), promotedSha256: sha256Buffer(promoted) };
}
@@ -318,46 +412,70 @@ export async function scanFleetDrift(): Promise<DriftEntry[]> {
}
export interface RunDriftInput {
servers?: string[]; // if omitted, scans the whole fleet
servers?: string[]; // if omitted, scans the whole fleet
dryRun?: boolean;
triggeredBy?: 'ui' | 'sweep' | 'cron';
expectedFingerprint?: string;
}
export class DriftPlanChanged extends Error {
constructor(public plan: DriftPlan) { super('The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'); }
constructor(public plan: DriftPlan) {
super(
'The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'
);
}
}
function driftTargets(servers?: string[]): string[] {
const registered = listServers().map(server => server.name);
if (servers && (!servers.length || servers.some(name => !registered.includes(name)))) throw new Error('Select at least one registered server. Refresh the fleet if a server was removed.');
const registered = listServers().map((server) => server.name);
if (servers && (!servers.length || servers.some((name) => !registered.includes(name))))
throw new Error(
'Select at least one registered server. Refresh the fleet if a server was removed.'
);
return [...new Set(servers ?? registered)].sort();
}
async function describePlan(servers: string[], entries: DriftEntry[]): Promise<DriftPlan> {
const files: DriftPlanFile[] = [];
for (const entry of entries) {
const before = await readFile(join(destOfKind(entry.server, entry.kind), entry.relPath)).catch((error: NodeJS.ErrnoException) => {
const before = await readFile(
join(destOfKind(entry.server, entry.kind), entry.relPath)
).catch((error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return null;
throw error;
});
const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry);
files.push({ server: entry.server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null });
const after =
entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry);
files.push({
server: entry.server,
relPath: entry.relPath,
kind: entry.kind,
driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null,
afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null,
afterSize: after?.byteLength ?? null
});
}
return buildPlan(servers, files);
}
function buildPlan(servers: string[], files: DriftPlanFile[]): DriftPlan {
files.sort((a, b) => `${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`));
return { servers, files, scannedAt: new Date().toISOString(), fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files }))) };
files.sort((a, b) =>
`${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`)
);
return {
servers,
files,
scannedAt: new Date().toISOString(),
fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files })))
};
}
export async function previewDrift(servers?: string[]): Promise<DriftPlan> {
const targets = driftTargets(servers);
const entries: DriftEntry[] = [];
for (const server of targets) entries.push(...await scanServerDrift(server));
for (const server of targets) entries.push(...(await scanServerDrift(server)));
return describePlan(targets, entries);
}
@@ -373,7 +491,10 @@ export interface RunDriftResult {
}
function deploymentIdNow(): string {
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
const stamp = new Date()
.toISOString()
.replace(/[-:]/g, '')
.replace(/\.\d+Z$/, 'Z');
return `dep-${stamp}-drift-${randomUUID().slice(0, 8)}`;
}
@@ -386,47 +507,103 @@ export interface PreparedDriftCampaign {
export async function prepareDriftCampaign(server: string): Promise<PreparedDriftCampaign> {
const servers = driftTargets([server]);
const entries = await scanServerDrift(server);
const files = await scanBatch(entries, async entry => {
const files = await scanBatch(entries, async (entry) => {
const before = await readExistingFile(join(destOfKind(server, entry.kind), entry.relPath));
const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry);
if (entry.driftKind !== 'untracked' && !after) throw new Error(`Cannot resolve promoted content for ${entry.relPath}`);
return { before, after, file: { server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } };
const after =
entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry);
if (entry.driftKind !== 'untracked' && !after)
throw new Error(`Cannot resolve promoted content for ${entry.relPath}`);
return {
before,
after,
file: {
server,
relPath: entry.relPath,
kind: entry.kind,
driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null,
afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null,
afterSize: after?.byteLength ?? null
}
};
});
if (entries.length) await requireServerDirectory(server);
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
return { plan: buildPlan(servers, files.map(({ file }) => file)), files };
if (hasActiveBeta(server))
throw new Error('Active beta owns this server; reconciliation deferred');
return {
plan: buildPlan(
servers,
files.map(({ file }) => file)
),
files
};
}
/** Record the audited plan atomically with a queued rebuild, never a second restart. */
export function createDriftRebuildCampaign(
server: string, operationId: string, incidentId: string | number, prepared: PreparedDriftCampaign
server: string,
operationId: string,
incidentId: string | number,
prepared: PreparedDriftCampaign
): string | null {
if (!prepared.files.length) return null;
if (prepared.plan.servers.length !== 1 || prepared.plan.servers[0] !== server
|| prepared.files.some(({ file }) => file.server !== server)) throw new Error('Drift evidence belongs to another server');
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
if (
prepared.plan.servers.length !== 1 ||
prepared.plan.servers[0] !== server ||
prepared.files.some(({ file }) => file.server !== server)
)
throw new Error('Drift evidence belongs to another server');
if (hasActiveBeta(server))
throw new Error('Active beta owns this server; reconciliation deferred');
return db().transaction(() => {
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'").get(operationId, server)) {
if (
!db()
.prepare(
"SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'"
)
.get(operationId, server)
) {
throw new Error('Drift campaign requires the matching rebuild operation');
}
const id = deploymentIdNow();
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')")
db()
.prepare(
"INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')"
)
.run(id, new Date().toISOString());
const insertFile = db().prepare(`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
for (const { file, before, after } of prepared.files) {
insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated',
file.relPath, file.beforeSha256, file.afterSha256, putFileBlob(before), putFileBlob(after));
insertFile.run(
id,
server,
after === null ? 'removed' : before === null ? 'added' : 'updated',
file.relPath,
file.beforeSha256,
file.afterSha256,
putFileBlob(before),
putFileBlob(after)
);
}
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId);
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)")
db()
.prepare(
`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
VALUES (?, ?, ?, 'pending', ?)`
)
.run(id, server, prepared.files.length, operationId);
db()
.prepare(
"INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)"
)
.run(operationId, id, server);
db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(id, `Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`);
db()
.prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(
id,
`Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`
);
emit({ type: 'deployment-opened', deploymentId: id, action: 'drift', servers: [server] });
return id;
})();
@@ -455,7 +632,12 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
}
// Recheck identities, existing bytes, and replacement bytes before any write.
const plan = input.expectedFingerprint ? await describePlan(scanTargets, perServer.flatMap(server => server.entries)) : null;
const plan = input.expectedFingerprint
? await describePlan(
scanTargets,
perServer.flatMap((server) => server.entries)
)
: null;
if (plan && plan.fingerprint !== input.expectedFingerprint) throw new DriftPlanChanged(plan);
const totalDrift = perServer.reduce((sum, s) => sum + s.entries.length, 0);
if (totalDrift === 0 || dryRun) {
@@ -464,7 +646,9 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
perServer: perServer.map((s) => ({
server: s.server,
removed: s.entries.filter((e) => e.driftKind === 'untracked').map((e) => e.relPath),
restored: s.entries.filter((e) => e.driftKind !== 'untracked').map((e) => e.relPath),
restored: s.entries
.filter((e) => e.driftKind !== 'untracked')
.map((e) => e.relPath),
skipped: []
})),
totalRemoved: perServer.reduce(
@@ -484,9 +668,11 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
// can be inserted as we go.
const depId = deploymentIdNow();
const now = new Date().toISOString();
db().prepare(
`INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)`
).run(depId, now, triggeredBy);
db()
.prepare(
`INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)`
)
.run(depId, now, triggeredBy);
const insertServer = db().prepare(
`INSERT INTO deployment_servers
(deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error)
@@ -511,53 +697,107 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
try {
await requireServerDirectory(s.server);
if (hasActiveBeta(s.server)) {
s.skipped.push(`${entry.relPath}: active beta owns this server; reconciliation deferred`);
s.skipped.push(
`${entry.relPath}: active beta owns this server; reconciliation deferred`
);
continue;
}
const reviewed = plan?.files.find(file => file.server === s.server && file.kind === entry.kind && file.relPath === entry.relPath);
const reviewed = plan?.files.find(
(file) =>
file.server === s.server &&
file.kind === entry.kind &&
file.relPath === entry.relPath
);
if (reviewed) {
const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; });
const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return null;
throw error;
});
if ((current ? sha256Buffer(current) : null) !== reviewed.beforeSha256) {
s.skipped.push(`${entry.relPath}: changed after review; scan and review again`);
s.skipped.push(
`${entry.relPath}: changed after review; scan and review again`
);
continue;
}
}
if (entry.driftKind === 'untracked') {
// Capture the reviewed deletion without touching the live tree.
const content = await readFile(abs);
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
if (hasActiveBeta(s.server))
throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, putFileBlob(content), null);
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) });
const file = insertFile.run(
depId,
s.server,
'removed',
entry.relPath,
sha256Buffer(content),
null,
putFileBlob(content),
null
);
queueFileChange({
sourceKind: 'campaign',
sourceId: depId,
server: s.server,
kind: entry.kind,
relPath: entry.relPath,
content: null,
inventorySource: `drift:${depId}`,
deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content)
});
})();
s.removed.push(entry.relPath);
} else {
// Persist exact replacement bytes so waiting cannot change the plan.
const written = await resolveManagedEntry(s.server, entry);
if (!written) {
s.skipped.push(`${entry.relPath}: missing artifact could not be resolved from the DB`);
s.skipped.push(
`${entry.relPath}: missing artifact could not be resolved from the DB`
);
continue;
}
if (reviewed && sha256Buffer(written) !== reviewed.afterSha256) {
s.skipped.push(`${entry.relPath}: promoted content changed after review; scan and review again`);
s.skipped.push(
`${entry.relPath}: promoted content changed after review; scan and review again`
);
continue;
}
const before = await readFile(abs).catch((err: NodeJS.ErrnoException) => {
if (err.code === 'ENOENT') return null;
throw err;
});
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
if (hasActiveBeta(s.server))
throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath,
before ? sha256Buffer(before) : null, sha256Buffer(written), putFileBlob(before), putFileBlob(written));
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: written,
scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed ? reviewed.beforeSha256 : before ? sha256Buffer(before) : null });
const file = insertFile.run(
depId,
s.server,
before ? 'updated' : 'added',
entry.relPath,
before ? sha256Buffer(before) : null,
sha256Buffer(written),
putFileBlob(before),
putFileBlob(written)
);
queueFileChange({
sourceKind: 'campaign',
sourceId: depId,
server: s.server,
kind: entry.kind,
relPath: entry.relPath,
content: written,
scriptVersionId:
entry.kind === 'config' ? null : (entry.scriptVersionId ?? null),
inventorySource: `drift:${depId}`,
deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed
? reviewed.beforeSha256
: before
? sha256Buffer(before)
: null
});
})();
s.restored.push(entry.relPath);
}
@@ -579,7 +819,10 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
return {
deploymentId: null,
perServer: perServer.map((s) => ({
server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
server: s.server,
removed: s.removed,
restored: s.restored,
skipped: s.skipped
})),
totalRemoved: 0,
totalRestored: 0,
@@ -598,7 +841,10 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
return {
deploymentId: depId,
perServer: perServer.map((s) => ({
server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
server: s.server,
removed: s.removed,
restored: s.restored,
skipped: s.skipped
})),
totalRemoved: perServer.reduce((sum, s) => sum + s.removed.length, 0),
totalRestored: perServer.reduce((sum, s) => sum + s.restored.length, 0),
+226 -69
View File
@@ -4,7 +4,11 @@ import { db } from '../db/index.js';
import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
import { nativeInstance, type NativeInstance } from '../backends/native.js';
import { nativeHealth } from '../backends/native.js';
import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js';
import {
rconStatusResult,
agentRconStatusResult,
type AgentRconReply
} from '../integrations/rcon.js';
import { withOperationSource } from '../hosts/operation-context.js';
import { getServer } from './servers.js';
import { latestGameSubmission } from './lifecycle-evidence.js';
@@ -14,8 +18,15 @@ import { ONLINE_MAX_MS } from '../../server-freshness.js';
const MAX_AGE_MS = 30_000;
const MAX_SERVERS = 1024;
type Probe = 'inspect' | 'rcon';
interface Observation<T> { result: T; observedAt: string }
interface Identity { key: string; port: number | null; checkedAt: string | null }
interface Observation<T> {
result: T;
observedAt: string;
}
interface Identity {
key: string;
port: number | null;
checkedAt: string | null;
}
interface Entry<T> {
identity: Identity;
startedAt: number;
@@ -28,21 +39,50 @@ export class MonitoringSkippedError extends Error {}
function identity(server: string, probe: Probe, snapshot?: MonitoringSnapshot): Identity {
const row = snapshot ? snapshot.get(server)?.server : getServer(server);
if (!row || row.destroyed_at) throw new MonitoringChangedError('Server is no longer registered; monitoring evidence is unavailable.');
const native = snapshot ? snapshot.get(server)?.native : row.native_instance_id ? nativeInstance(server) : null;
if (row.native_instance_id && (!native || (snapshot ? snapshot.get(server)?.host : getHost(native.host_id))?.connectionState !== 'online')) {
if (!row || row.destroyed_at)
throw new MonitoringChangedError(
'Server is no longer registered; monitoring evidence is unavailable.'
);
const native = snapshot
? snapshot.get(server)?.native
: row.native_instance_id
? nativeInstance(server)
: null;
if (
row.native_instance_id &&
(!native ||
(snapshot ? snapshot.get(server)?.host : getHost(native.host_id))?.connectionState !==
'online')
) {
throw new Error('Host agent is offline; monitoring evidence is unavailable.');
}
// Stop requests also invalidate observations, even though they do not
// establish a new game-telemetry epoch in latestGameSubmission().
const savedLifecycle = snapshot?.get(server)?.lifecycle;
const lifecycle = snapshot ? savedLifecycle ? { id: savedLifecycle.id, action: savedLifecycle.action, submitted_at: savedLifecycle.submitted_at } : null : db().prepare(`SELECT id, action, submitted_at FROM server_deployments
WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) ?? null;
const lifecycle = snapshot
? savedLifecycle
? {
id: savedLifecycle.id,
action: savedLifecycle.action,
submitted_at: savedLifecycle.submitted_at
}
: null
: (db()
.prepare(
`SELECT id, action, submitted_at FROM server_deployments
WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`
)
.get(server) ?? null);
return {
key: JSON.stringify([row.created_at, row.desired_state, snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server), lifecycle,
key: JSON.stringify([
row.created_at,
row.desired_state,
snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server),
lifecycle,
native ? [native.host_id, native.id, native.generation] : null,
probe === 'rcon' ? [row.rcon_host, config.rcon] : null]),
port: probe === 'rcon' ? native?.port ?? row.game_port : null,
probe === 'rcon' ? [row.rcon_host, config.rcon] : null
]),
port: probe === 'rcon' ? (native?.port ?? row.game_port) : null,
checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at
};
}
@@ -58,14 +98,32 @@ function reusable(startedAt: number): boolean {
return age >= 0 && age < MAX_AGE_MS;
}
async function observe<T extends { ok: boolean }>(server: string, fresh: boolean, probe: Probe,
entries: Map<string, Entry<T>>, request: (server: string) => Promise<T>, snapshot?: MonitoringSnapshot, batch = false): Promise<Observation<T>> {
async function observe<T extends { ok: boolean }>(
server: string,
fresh: boolean,
probe: Probe,
entries: Map<string, Entry<T>>,
request: (server: string) => Promise<T>,
snapshot?: MonitoringSnapshot,
batch = false
): Promise<Observation<T>> {
let current: Identity;
try { current = identity(server, probe, snapshot); }
catch (error) { entries.delete(server); throw error; }
try {
current = identity(server, probe, snapshot);
} catch (error) {
entries.delete(server);
throw error;
}
const saved = entries.get(server);
const savedAt = saved?.observation ? Date.parse(saved.observation.observedAt) : saved?.startedAt ?? 0;
if (!fresh && saved && sameIdentity(saved.identity, current) && !newerEvidence(current, savedAt)) {
const savedAt = saved?.observation
? Date.parse(saved.observation.observedAt)
: (saved?.startedAt ?? 0);
if (
!fresh &&
saved &&
sameIdentity(saved.identity, current) &&
!newerEvidence(current, savedAt)
) {
if (saved.pending) return saved.pending;
if (saved.observation && reusable(saved.startedAt)) return saved.observation;
}
@@ -76,22 +134,27 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
entries.delete(server);
entries.set(server, entry);
while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!);
const pending = Promise.resolve().then(() => request(server)).then(result => {
if (!batch) snapshot?.refresh(server);
const latest = identity(server, probe, snapshot);
if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) {
throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.');
}
const observation = { result, observedAt: new Date().toISOString() };
if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) {
entry.identity = latest;
entry.observation = observation;
}
return observation;
}).finally(() => {
entry.pending = undefined;
if (entries.get(server) === entry && !entry.observation) entries.delete(server);
});
const pending = Promise.resolve()
.then(() => request(server))
.then((result) => {
if (!batch) snapshot?.refresh(server);
const latest = identity(server, probe, snapshot);
if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) {
throw new MonitoringChangedError(
'Server changed during monitoring; the previous observation was discarded.'
);
}
const observation = { result, observedAt: new Date().toISOString() };
if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) {
entry.identity = latest;
entry.observation = observation;
}
return observation;
})
.finally(() => {
entry.pending = undefined;
if (entries.get(server) === entry && !entry.observation) entries.delete(server);
});
entry.pending = pending;
return pending;
}
@@ -112,14 +175,19 @@ export function monitoringIdentity(server: string, snapshot?: MonitoringSnapshot
const container = identity(server, 'inspect', snapshot);
const game = identity(server, 'rcon', snapshot);
// Identity includes connection configuration; persist only its fingerprint.
return createHash('sha256').update(JSON.stringify([container.key, game.key, game.port])).digest('hex');
return createHash('sha256')
.update(JSON.stringify([container.key, game.key, game.port]))
.digest('hex');
}
/** Capture ownership before a status pass; publish only evidence for that instance. */
export function monitoringStatusWriter(server: string, snapshot?: MonitoringSnapshot) {
let before: string;
try { before = identity(server, 'inspect', snapshot).key; }
catch { return () => {}; }
try {
before = identity(server, 'inspect', snapshot).key;
} catch {
return () => {};
}
const slot = { snapshot: statuses.get(server)?.snapshot };
statuses.delete(server);
statuses.set(server, slot);
@@ -133,12 +201,17 @@ export function monitoringStatusWriter(server: string, snapshot?: MonitoringSnap
const current = slot.snapshot;
if (current && current.container.observedAt > container.observedAt) return;
slot.snapshot = { identity: monitoringIdentity(server, snapshot), container, game };
} catch { statuses.delete(server); }
} catch {
statuses.delete(server);
}
};
}
/** Latest completed status-monitor pass, with stale or superseded evidence excluded. */
export function monitoringStatus(server: string, snapshot?: MonitoringSnapshot): MonitoringStatus | null {
export function monitoringStatus(
server: string,
snapshot?: MonitoringSnapshot
): MonitoringStatus | null {
const saved = statuses.get(server)?.snapshot;
if (!saved) return null;
const fresh = (at: string) => {
@@ -146,43 +219,94 @@ export function monitoringStatus(server: string, snapshot?: MonitoringSnapshot):
return age >= 0 && age <= ONLINE_MAX_MS;
};
try {
if (saved.identity !== monitoringIdentity(server, snapshot)
|| !fresh(saved.container.observedAt)
|| newerEvidence(identity(server, 'inspect', snapshot), Date.parse(saved.container.observedAt))) return null;
const game = saved.game && fresh(saved.game.observedAt)
&& !newerEvidence(identity(server, 'rcon', snapshot), Date.parse(saved.game.observedAt)) ? saved.game : undefined;
if (
saved.identity !== monitoringIdentity(server, snapshot) ||
!fresh(saved.container.observedAt) ||
newerEvidence(
identity(server, 'inspect', snapshot),
Date.parse(saved.container.observedAt)
)
)
return null;
const game =
saved.game &&
fresh(saved.game.observedAt) &&
!newerEvidence(identity(server, 'rcon', snapshot), Date.parse(saved.game.observedAt))
? saved.game
: undefined;
return { ...saved, game };
} catch { return null; }
} catch {
return null;
}
}
/** Keep the actual sample time: a later heartbeat must not renew old evidence. */
export function heartbeatInspection(server: string, snapshot?: MonitoringSnapshot): Observation<Awaited<ReturnType<typeof nativeHealth>>> | null {
export function heartbeatInspection(
server: string,
snapshot?: MonitoringSnapshot
): Observation<Awaited<ReturnType<typeof nativeHealth>>> | null {
const row = snapshot ? snapshot.get(server)?.server : getServer(server);
if (!row?.native_instance_id || !row.host_id) return null;
const host = snapshot ? snapshot.get(server)?.host : getHost(row.host_id);
if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state')) return null;
const state = host.inventory.instanceStates?.find(state => state.instanceId === row.native_instance_id && state.generation === row.instance_generation);
if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state'))
return null;
const state = host.inventory.instanceStates?.find(
(state) =>
state.instanceId === row.native_instance_id &&
state.generation === row.instance_generation
);
if (!state) return null;
const at = Date.parse(state.observedAt);
const lifecycle = snapshot ? { at: snapshot.get(server)?.lifecycleAt } : db().prepare(`SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at
FROM server_deployments WHERE server=? AND server_deleted_at IS NULL`).get(server) as { at: string | null };
if (!reusable(at) || at < Date.parse(row.created_at) || at < Date.parse(lifecycle.at ?? '')
|| at < Date.parse((snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server)) ?? '') || at < Date.parse(row.container_checked_at ?? '')) return null;
return { observedAt: new Date(at).toISOString(), result: state.healthy
? { ok: true, state: state.state } : { ok: false, state: state.state, reason: `Container ${state.state}` } };
const lifecycle = snapshot
? { at: snapshot.get(server)?.lifecycleAt }
: (db()
.prepare(
`SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at
FROM server_deployments WHERE server=? AND server_deleted_at IS NULL`
)
.get(server) as { at: string | null });
if (
!reusable(at) ||
at < Date.parse(row.created_at) ||
at < Date.parse(lifecycle.at ?? '') ||
at <
Date.parse(
(snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server)) ?? ''
) ||
at < Date.parse(row.container_checked_at ?? '')
)
return null;
return {
observedAt: new Date(at).toISOString(),
result: state.healthy
? { ok: true, state: state.state }
: { ok: false, state: state.state, reason: `Container ${state.state}` }
};
}
type RconObservation = Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
/** Prime a whole host before the per-host queue admits just its first two servers. */
export function prepareMonitoringRcon(servers: string[], snapshot?: MonitoringSnapshot): Map<string, () => Promise<RconObservation>> {
export function prepareMonitoringRcon(
servers: string[],
snapshot?: MonitoringSnapshot
): Map<string, () => Promise<RconObservation>> {
const probes = new Map<string, () => Promise<RconObservation>>();
const batches = new Map<string, { instances: NativeInstance[]; result: Promise<{ results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> }> }>();
const batches = new Map<
string,
{
instances: NativeInstance[];
result: Promise<{
results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }>;
}>;
}
>();
for (const server of servers) {
const instance = snapshot ? snapshot.get(server)?.native : nativeInstance(server);
if (!instance) continue;
const host = snapshot ? snapshot.get(server)?.host : getHost(instance.host_id);
if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch')) continue;
if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch'))
continue;
const container = heartbeatInspection(server, snapshot);
if (container && !container.result.ok && container.result.state !== 'unknown') continue;
const before = identity(server, 'rcon', snapshot);
@@ -193,40 +317,73 @@ export function prepareMonitoringRcon(servers: string[], snapshot?: MonitoringSn
// observe() invokes each request in the same microtask turn;
// enqueue afterward so cached or shared probes need no batch slot.
const result = Promise.resolve().then(async () => {
const operation = withOperationSource('Status monitor', () => queueHostOperation(instance.host_id, 'rcon.status-batch', {
instances: instances.map(value => ({ instanceId: value.id, generation: value.generation }))
}));
const operation = withOperationSource('Status monitor', () =>
queueHostOperation(instance.host_id, 'rcon.status-batch', {
instances: instances.map((value) => ({
instanceId: value.id,
generation: value.generation
}))
})
);
// Allow delivery and receipt overhead beyond the agent's 20-second budget.
const receipt = await waitHostOperationStatus(operation.id, 25_000);
snapshot?.refresh();
if (receipt.state !== 'completed') throw new Error(receipt.error ?? 'Host status batch did not complete.');
return receipt.result as { results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> };
if (receipt.state !== 'completed')
throw new Error(receipt.error ?? 'Host status batch did not complete.');
return receipt.result as {
results: Array<{
instanceId: string;
generation: number;
reply: AgentRconReply;
}>;
};
});
batch = { instances, result }; batches.set(instance.host_id, batch);
batch = { instances, result };
batches.set(instance.host_id, batch);
}
batch.instances.push(instance);
const endpoint = { host: `agent:${instance.host_id}`, port: instance.port };
try {
const receipt = await batch.result;
const replies = receipt.results.filter(value => value.instanceId === instance.id && value.generation === instance.generation);
if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string') throw new Error('Host status batch omitted a valid instance result.');
if (replies[0].reply.reason === 'deadline') throw new MonitoringSkippedError('Host status batch budget expired before this server was checked.');
const replies = receipt.results.filter(
(value) =>
value.instanceId === instance.id && value.generation === instance.generation
);
if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string')
throw new Error('Host status batch omitted a valid instance result.');
if (replies[0].reply.reason === 'deadline')
throw new MonitoringSkippedError(
'Host status batch budget expired before this server was checked.'
);
return agentRconStatusResult(replies[0].reply, endpoint);
} catch (error) {
if (error instanceof MonitoringSkippedError) throw error;
return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message };
return {
ok: false as const,
endpoint,
reason: 'timeout' as const,
detail: (error as Error).message
};
}
};
// Observe before dispatch, including servers still waiting for their
// queue slot. Consume failures too, so a failed batch is never retried
// once per server or left as an unhandled rejected promise.
const pending = observe(server, false, 'rcon', rcon, request, snapshot, true).then(value => ({ value }), error => ({ error }));
const pending = observe(server, false, 'rcon', rcon, request, snapshot, true).then(
(value) => ({ value }),
(error) => ({ error })
);
probes.set(server, async () => {
const result = await pending;
if ('error' in result) throw result.error;
const current = identity(server, 'rcon', snapshot);
if (!sameIdentity(before, current) || newerEvidence(current, Date.parse(result.value.observedAt))) {
throw new MonitoringChangedError('Server changed before consuming the host status batch.');
if (
!sameIdentity(before, current) ||
newerEvidence(current, Date.parse(result.value.observedAt))
) {
throw new MonitoringChangedError(
'Server changed before consuming the host status batch.'
);
}
return result.value;
});
+7 -2
View File
@@ -9,9 +9,14 @@ const STALE_MAX_MS = 120_000;
* `stale` on a blocking RCON failure or if the last reply is >90-120s old.
* `online` if fresh, including the allowance for polling and delivery.
*/
export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
export function classifyHealth(
lastSeenAt: string | null,
lastError: string | null,
containerState?: string | null
): Health {
if (containerState === 'starting' || containerState === 'restarting') return 'starting';
if (containerState && containerState !== 'running' && containerState !== 'unknown') return 'offline';
if (containerState && containerState !== 'running' && containerState !== 'unknown')
return 'offline';
if (!lastSeenAt) return 'offline';
const age = Date.now() - new Date(lastSeenAt).getTime();
if (freshStatusTimestamp(lastSeenAt, Date.now())) {
+17 -5
View File
@@ -2,23 +2,35 @@ import { db } from '../db/index.js';
import { destroyServer } from '../provisioning/index.js';
export function housePartyTargets(): string[] {
return (db().prepare(`
return (
db()
.prepare(
`
SELECT name FROM servers WHERE destroyed_at IS NULL
UNION
SELECT server AS name FROM server_deployments
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
ORDER BY name
`).all() as { name: string }[]).map(row => row.name);
`
)
.all() as { name: string }[]
).map((row) => row.name);
}
export function queueHouseParty(expectedServers: string[]) {
return db().transaction(() => {
const servers = housePartyTargets();
const expected = [...expectedServers].sort();
if (servers.length !== expected.length || servers.some((name, index) => name !== expected[index])) {
return { ok: false as const, error: 'The fleet changed. Refresh the preview and review the servers again.' };
if (
servers.length !== expected.length ||
servers.some((name, index) => name !== expected[index])
) {
return {
ok: false as const,
error: 'The fleet changed. Refresh the preview and review the servers again.'
};
}
const deployments = servers.map(server => {
const deployments = servers.map((server) => {
const result = destroyServer(server, 'house-party-protocol');
if (!result.ok) throw new Error(result.message);
return { server, deploymentId: result.deploymentId! };
+3 -1
View File
@@ -36,7 +36,9 @@ export function recordScript(
}
export function removeScript(server: string, relPath: string): void {
db().prepare('DELETE FROM server_scripts WHERE server = ? AND rel_path = ?').run(server, relPath);
db()
.prepare('DELETE FROM server_scripts WHERE server = ? AND rel_path = ?')
.run(server, relPath);
}
export function getServerScript(server: string, relPath: string): ServerScriptRow | undefined {
+248 -68
View File
@@ -1,12 +1,20 @@
import { packFileJson, unpackFileJson } from '../db/file-blobs.js';
import { randomUUID } from 'node:crypto';
import type { BlueprintSummary, LaunchRecipeSummary, LaunchFileSummary } from '$lib/deployment-plan.js';
import type {
BlueprintSummary,
LaunchRecipeSummary,
LaunchFileSummary
} from '$lib/deployment-plan.js';
import { config, configFileForGame } from '../config.js';
import { db } from '../db/index.js';
import { zombieMapBySlug, type ZombieGame } from './map-aliases.js';
import { promotedArtifactMetadata, promotedArtifacts } from './promoted-artifacts.js';
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
import { getBaselineForScope, getVersionMetadata, listPromotedOverlaysForScope } from './scripts.js';
import {
getBaselineForScope,
getVersionMetadata,
listPromotedOverlaysForScope
} from './scripts.js';
import { sha256Buffer } from './files.js';
export interface LaunchRecipePlan {
@@ -28,165 +36,337 @@ export interface LaunchRecipe extends LaunchRecipePlan {
files: (LaunchFileSummary & { content: string })[];
}
export interface BlueprintEntry { count: number; recipe: LaunchRecipePlan }
interface BlueprintSelection { count: number; game: ZombieGame; mapSlug: string }
interface BlueprintRow { id: string; name: string; description: string; created_at: string; recipe: string }
export interface BlueprintEntry {
count: number;
recipe: LaunchRecipePlan;
}
interface BlueprintSelection {
count: number;
game: ZombieGame;
mapSlug: string;
}
interface BlueprintRow {
id: string;
name: string;
description: string;
created_at: string;
recipe: string;
}
/** Legacy snapshots provide the lineup only; their file versions never govern a new launch. */
function readEntries(raw: string): BlueprintSelection[] {
const saved = JSON.parse(raw) as LaunchRecipe | { format: 2; entries: BlueprintEntry[] }
const saved = JSON.parse(raw) as
| LaunchRecipe
| { format: 2; entries: BlueprintEntry[] }
| { format: 3; entries: BlueprintSelection[] };
if (saved.format === 3) return saved.entries;
const entries = saved.format === 2 ? saved.entries : [{ count: 1, recipe: saved }];
return entries.map(({ count, recipe }) => ({ count, game: recipe.game, mapSlug: recipe.mapSlug }));
return entries.map(({ count, recipe }) => ({
count,
game: recipe.game,
mapSlug: recipe.mapSlug
}));
}
function summarizeBlueprint(row: BlueprintRow): BlueprintSummary {
const entries = readEntries(row.recipe).map(entry => ({ ...entry,
mapName: zombieMapBySlug(entry.game, entry.mapSlug)?.friendly ?? entry.mapSlug }));
return { id: row.id, name: row.name, description: row.description, createdAt: row.created_at,
entries, totalServers: entries.reduce((total, entry) => total + entry.count, 0) };
const entries = readEntries(row.recipe).map((entry) => ({
...entry,
mapName: zombieMapBySlug(entry.game, entry.mapSlug)?.friendly ?? entry.mapSlug
}));
return {
id: row.id,
name: row.name,
description: row.description,
createdAt: row.created_at,
entries,
totalServers: entries.reduce((total, entry) => total + entry.count, 0)
};
}
export function blueprintEntries(id: string): { entries: BlueprintEntry[]; blueprintName: string } {
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as BlueprintRow | undefined;
if (!row) throw new Error('This collection is no longer available. Choose another starting point.');
const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipePlan(game, mapSlug) }));
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as
BlueprintRow | undefined;
if (!row)
throw new Error('This collection is no longer available. Choose another starting point.');
const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({
count,
recipe: captureRecipePlan(game, mapSlug)
}));
return { entries, blueprintName: row.name };
}
function buildEntries(input: unknown): BlueprintSelection[] {
if (!Array.isArray(input) || input.length < 1 || input.length > 40) throw new Error('Add between 1 and 40 map selections.');
if (!Array.isArray(input) || input.length < 1 || input.length > 40)
throw new Error('Add between 1 and 40 map selections.');
const seen = new Set<string>();
const entries = input.map(entry => {
if (!entry || (entry.game !== 't5' && entry.game !== 't6') || typeof entry.mapSlug !== 'string'
|| !zombieMapBySlug(entry.game, entry.mapSlug)) throw new Error('Each selection needs a supported game and map.');
if (!Number.isSafeInteger(entry.count) || entry.count < 1 || entry.count > 15) throw new Error('Choose 1–15 lobbies for each map.');
const entries = input.map((entry) => {
if (
!entry ||
(entry.game !== 't5' && entry.game !== 't6') ||
typeof entry.mapSlug !== 'string' ||
!zombieMapBySlug(entry.game, entry.mapSlug)
)
throw new Error('Each selection needs a supported game and map.');
if (!Number.isSafeInteger(entry.count) || entry.count < 1 || entry.count > 15)
throw new Error('Choose 1–15 lobbies for each map.');
const key = `${entry.game}:${entry.mapSlug}`;
if (seen.has(key)) throw new Error('Each game/map can appear once. Increase its lobby count instead.');
if (seen.has(key))
throw new Error('Each game/map can appear once. Increase its lobby count instead.');
seen.add(key);
return { count: entry.count as number, game: entry.game as ZombieGame, mapSlug: entry.mapSlug as string };
return {
count: entry.count as number,
game: entry.game as ZombieGame,
mapSlug: entry.mapSlug as string
};
});
if (entries.reduce((total, entry) => total + entry.count, 0) > 100) throw new Error('A collection can launch at most 100 servers.');
if (entries.reduce((total, entry) => total + entry.count, 0) > 100)
throw new Error('A collection can launch at most 100 servers.');
return entries;
}
export function captureRecipe(game: ZombieGame, mapSlug: string, rconPassword?: string): LaunchRecipe {
export function captureRecipe(
game: ZombieGame,
mapSlug: string,
rconPassword?: string
): LaunchRecipe {
return buildRecipe(game, mapSlug, true, rconPassword);
}
/** Preview and admission keep file references; execution captures the current bytes. */
export function captureRecipePlan(game: ZombieGame, mapSlug: string, rconPassword?: string): LaunchRecipePlan {
export function captureRecipePlan(
game: ZombieGame,
mapSlug: string,
rconPassword?: string
): LaunchRecipePlan {
return buildRecipe(game, mapSlug, false, rconPassword);
}
function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: true, rconPassword?: string): LaunchRecipe;
function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: false, rconPassword?: string): LaunchRecipePlan;
function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean, rconPassword?: string): LaunchRecipePlan {
function buildRecipe(
game: ZombieGame,
mapSlug: string,
includeContent: true,
rconPassword?: string
): LaunchRecipe;
function buildRecipe(
game: ZombieGame,
mapSlug: string,
includeContent: false,
rconPassword?: string
): LaunchRecipePlan;
function buildRecipe(
game: ZombieGame,
mapSlug: string,
includeContent: boolean,
rconPassword?: string
): LaunchRecipePlan {
const map = zombieMapBySlug(game, mapSlug);
if (!map) throw new Error('Choose a supported game and map.');
const metadata = ({ kind, script, version }: ReturnType<typeof promotedArtifactMetadata>[number]): LaunchFileSummary => ({
name: script.name, kind, path: script.target_rel_path, versionId: version.id,
version: version.version, sha256: version.sha256, size: version.size
const metadata = ({
kind,
script,
version
}: ReturnType<typeof promotedArtifactMetadata>[number]): LaunchFileSummary => ({
name: script.name,
kind,
path: script.target_rel_path,
versionId: version.id,
version: version.version,
sha256: version.sha256,
size: version.size
});
const files: LaunchRecipePlan['files'] = includeContent
? promotedArtifacts(game, map.canonical).map(artifact => ({ ...metadata(artifact), content: artifact.version.content.toString('base64') }))
? promotedArtifacts(game, map.canonical).map((artifact) => ({
...metadata(artifact),
content: artifact.version.content.toString('base64')
}))
: promotedArtifactMetadata(game, map.canonical).map(metadata);
const addConfig = (name: string, path: string, content: Buffer) => files.push({
name, path, kind: 'config', versionId: null, version: null,
sha256: sha256Buffer(content), size: content.length, ...(includeContent ? { content: content.toString('base64') } : {})
});
const addConfig = (name: string, path: string, content: Buffer) =>
files.push({
name,
path,
kind: 'config',
versionId: null,
version: null,
sha256: sha256Buffer(content),
size: content.length,
...(includeContent ? { content: content.toString('base64') } : {})
});
const composed = getPromotedServerConfig(game, map.canonical, map.slug, rconPassword);
let configSource = 'Plutainer defaults, seeded on first boot';
if (composed) {
const baseline = getBaselineForScope('game', game)!;
const layers = [baseline, ...listPromotedOverlaysForScope('game', game), ...listPromotedOverlaysForScope('map', map.canonical)];
configSource = layers.map(layer => `${layer.name} v${getVersionMetadata(layer.current_version_id!)!.version}`).join(' + ');
const layers = [
baseline,
...listPromotedOverlaysForScope('game', game),
...listPromotedOverlaysForScope('map', map.canonical)
];
configSource = layers
.map(
(layer) =>
`${layer.name} v${getVersionMetadata(layer.current_version_id!)!.version}`
)
.join(' + ');
addConfig('Composed server configuration', configFileForGame(game), composed);
}
addConfig('Selected map', MAP_CONFIG_FILE, getServerMapConfig(game, map.canonical, map.slug)!);
return { format: 1, game, mapSlug, mapName: map.friendly, expectedMap: map.canonical,
capturedAt: new Date().toISOString(), image: config.defaultRuntimeImage,
configFile: configFileForGame(game), configSource, files };
return {
format: 1,
game,
mapSlug,
mapName: map.friendly,
expectedMap: map.canonical,
capturedAt: new Date().toISOString(),
image: config.defaultRuntimeImage,
configFile: configFileForGame(game),
configSource,
files
};
}
export function summarizeRecipe(recipe: LaunchRecipePlan, fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe)))): LaunchRecipeSummary {
export function summarizeRecipe(
recipe: LaunchRecipePlan,
fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe)))
): LaunchRecipeSummary {
const { format: _format, files, ...summary } = recipe;
return { ...summary, files: files.map(({ content: _content, ...file }) => file),
fingerprint };
return { ...summary, files: files.map(({ content: _content, ...file }) => file), fingerprint };
}
/** Keep the small detail response beside the immutable attempted file bytes. */
export function recipeStorage(recipe: LaunchRecipePlan): { recipe: string; recipeSummary: string } {
const encoded = JSON.stringify(recipe);
return { recipe: JSON.stringify(packFileJson(recipe).value), recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) };
return {
recipe: JSON.stringify(packFileJson(recipe).value),
recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded))))
};
}
export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
const row = db().prepare('SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
export function recipeSummaryForDeployment(
id: string
): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
const row = db()
.prepare(
'SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?'
)
.get(id) as { recipe_summary: string | null; blueprint_name: string | null } | undefined;
if (!row) return null;
if (row.recipe_summary !== null) return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name };
if (row.recipe_summary !== null)
return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name };
// Old snapshots remain readable. Populate on first access, without making
// controller startup scan and decode every retained plugin in its history.
const launch = recipeForDeployment(id)!;
const summary = summarizeRecipe(launch.recipe);
db().prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?').run(JSON.stringify(summary), id);
db()
.prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?')
.run(JSON.stringify(summary), id);
return { recipe: summary, blueprintName: row.blueprint_name };
}
export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
const row = db().prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
export function recipeForDeployment(
id: string
): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
const row = db()
.prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
.get(id) as { recipe: string; blueprint_name: string | null } | undefined;
return row ? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name } : null;
return row
? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name }
: null;
}
export function blueprintRecipe(id: string): { recipe: LaunchRecipePlan; blueprintName: string } {
const { entries, blueprintName } = blueprintEntries(id);
if (entries.length !== 1 || entries[0].count !== 1) throw new Error('This collection launches multiple servers. Use the collection preview.');
if (entries.length !== 1 || entries[0].count !== 1)
throw new Error('This collection launches multiple servers. Use the collection preview.');
return { recipe: entries[0].recipe, blueprintName };
}
export function listBlueprints(): BlueprintSummary[] {
return (db().prepare('SELECT * FROM server_blueprints ORDER BY name COLLATE NOCASE').all() as BlueprintRow[]).map(summarizeBlueprint);
return (
db()
.prepare('SELECT * FROM server_blueprints ORDER BY name COLLATE NOCASE')
.all() as BlueprintRow[]
).map(summarizeBlueprint);
}
function blueprintName(input: { name: unknown; description?: unknown }, exceptId = ''): string {
if (typeof input.name !== 'string' || !input.name.trim() || input.name.trim().length > 80) throw new Error('Give the collection a name of 1–80 characters.');
if (input.description !== undefined && (typeof input.description !== 'string' || input.description.length > 500)) throw new Error('Keep the description to 500 characters.');
if (db().prepare('SELECT id FROM server_blueprints WHERE name = ? COLLATE NOCASE AND id != ?').get(input.name.trim(), exceptId)) throw new Error('A collection with this name already exists. Choose a new name.');
if (typeof input.name !== 'string' || !input.name.trim() || input.name.trim().length > 80)
throw new Error('Give the collection a name of 1–80 characters.');
if (
input.description !== undefined &&
(typeof input.description !== 'string' || input.description.length > 500)
)
throw new Error('Keep the description to 500 characters.');
if (
db()
.prepare('SELECT id FROM server_blueprints WHERE name = ? COLLATE NOCASE AND id != ?')
.get(input.name.trim(), exceptId)
)
throw new Error('A collection with this name already exists. Choose a new name.');
return input.name.trim();
}
export function saveBlueprint(input: { name: unknown; description?: unknown; previewId?: unknown; deploymentId?: unknown; entries?: unknown }): BlueprintSummary {
export function saveBlueprint(input: {
name: unknown;
description?: unknown;
previewId?: unknown;
deploymentId?: unknown;
entries?: unknown;
}): BlueprintSummary {
const name = blueprintName(input);
if ([input.previewId, input.deploymentId, input.entries].filter(value => value !== undefined).length !== 1) throw new Error('Choose exactly one launch preview, recorded deployment, or set of maps.');
if (
[input.previewId, input.deploymentId, input.entries].filter((value) => value !== undefined)
.length !== 1
)
throw new Error('Choose exactly one launch preview, recorded deployment, or set of maps.');
let recipe: LaunchRecipePlan | undefined;
if (typeof input.previewId === 'string') {
const row = db().prepare('SELECT recipe FROM server_launch_previews WHERE id = ? AND expires_at > ?')
const row = db()
.prepare('SELECT recipe FROM server_launch_previews WHERE id = ? AND expires_at > ?')
.get(input.previewId, new Date().toISOString()) as { recipe: string } | undefined;
if (row) recipe = JSON.parse(row.recipe);
} else if (typeof input.deploymentId === 'string') recipe = recipeForDeployment(input.deploymentId)?.recipe;
if (!recipe && input.entries === undefined) throw new Error('The launch recipe is unavailable or the preview expired. Generate a fresh preview.');
const entries = input.entries !== undefined ? buildEntries(input.entries)
: buildEntries([{ count: 1, game: recipe!.game, mapSlug: recipe!.mapSlug }]);
} else if (typeof input.deploymentId === 'string')
recipe = recipeForDeployment(input.deploymentId)?.recipe;
if (!recipe && input.entries === undefined)
throw new Error(
'The launch recipe is unavailable or the preview expired. Generate a fresh preview.'
);
const entries =
input.entries !== undefined
? buildEntries(input.entries)
: buildEntries([{ count: 1, game: recipe!.game, mapSlug: recipe!.mapSlug }]);
const id = randomUUID();
const createdAt = new Date().toISOString();
const saved = JSON.stringify({ format: 3, entries });
db().prepare('INSERT INTO server_blueprints (id, name, description, created_at, recipe) VALUES (?, ?, ?, ?, ?)')
db()
.prepare(
'INSERT INTO server_blueprints (id, name, description, created_at, recipe) VALUES (?, ?, ?, ?, ?)'
)
.run(id, name, input.description ?? '', createdAt, saved);
return summarizeBlueprint({ id, name, description: (input.description as string) ?? '', created_at: createdAt, recipe: saved });
return summarizeBlueprint({
id,
name,
description: (input.description as string) ?? '',
created_at: createdAt,
recipe: saved
});
}
export function updateBlueprint(id: string, input: { name: unknown; description?: unknown; entries?: unknown }): BlueprintSummary {
export function updateBlueprint(
id: string,
input: { name: unknown; description?: unknown; entries?: unknown }
): BlueprintSummary {
return db().transaction(() => {
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as BlueprintRow | undefined;
const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as
BlueprintRow | undefined;
if (!row) throw new Error('Collection not found.');
const name = blueprintName(input, id);
const entries = buildEntries(input.entries);
const recipe = JSON.stringify({ format: 3, entries });
const description = (input.description as string) ?? '';
db().prepare('UPDATE server_blueprints SET name = ?, description = ?, recipe = ? WHERE id = ?').run(name, description, recipe, id);
db()
.prepare(
'UPDATE server_blueprints SET name = ?, description = ?, recipe = ? WHERE id = ?'
)
.run(name, description, recipe, id);
return summarizeBlueprint({ ...row, name, description, recipe });
})();
}
+133 -37
View File
@@ -20,7 +20,9 @@ interface Participation {
}
function mapIdentity(handle: Database.Database, report: GameTelemetry) {
const server = handle.prepare('SELECT game, map, map_slug FROM servers WHERE name = ?').get(report.server) as
const server = handle
.prepare('SELECT game, map, map_slug FROM servers WHERE name = ?')
.get(report.server) as
{ game: string | null; map: string | null; map_slug: string | null } | undefined;
const parsed = parseServerName(report.server);
const game = (server?.game ?? parsed?.game ?? 'unknown').toLowerCase();
@@ -28,15 +30,22 @@ function mapIdentity(handle: Database.Database, report: GameTelemetry) {
// A shared engine map (notably zm_transit) is not enough to identify T6's
// location/ruleset. Use the registered selection only when it matches.
const parsedSlug = parsed?.mapCanonical === map ? parsed.mapSlug : undefined;
const slug = server?.map === map ? server.map_slug ?? parsedSlug : parsedSlug;
const candidates = zombieMaps(game).filter(entry => entry.canonical === map);
const selected = game === 't6' && candidates.length > 1 && !slug
? undefined : selectedZombieMap(game, map, slug);
const variant = game === 't6' ? selected?.slug ?? slug ?? 'unspecified' : '';
const slug = server?.map === map ? (server.map_slug ?? parsedSlug) : parsedSlug;
const candidates = zombieMaps(game).filter((entry) => entry.canonical === map);
const selected =
game === 't6' && candidates.length > 1 && !slug
? undefined
: selectedZombieMap(game, map, slug);
const variant = game === 't6' ? (selected?.slug ?? slug ?? 'unspecified') : '';
return {
game, map,
game,
map,
key: [game, map, ...(variant ? [variant] : [])].join(':'),
name: selected?.friendly ?? (game === 't6' && candidates.length > 1 ? `${map} (${slug ?? 'unspecified variant'})` : map)
name:
selected?.friendly ??
(game === 't6' && candidates.length > 1
? `${map} (${slug ?? 'unspecified variant'})`
: map)
};
}
@@ -44,48 +53,92 @@ function fingerprint(report: GameTelemetry): string {
// Receipt timestamps and roster ordering do not identify new gameplay.
const { receivedAt: _receivedAt, timing: _timing, ...event } = report;
const players = [...event.players].sort((a, b) => a.guid.localeCompare(b.guid));
return createHash('sha256').update(JSON.stringify({ ...event, players })).digest('hex');
return createHash('sha256')
.update(JSON.stringify({ ...event, players }))
.digest('hex');
}
/** Called inside the same transaction as the latest scoreboard snapshot. */
export function recordLeaderboardTelemetry(handle: Database.Database, report: GameTelemetry): void {
const identity = mapIdentity(handle, report);
const signature = fingerprint(report);
let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1')
let match = handle
.prepare(
'SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1'
)
.get(report.server) as Match | undefined;
if (match?.last_fingerprint === signature && match.map_key === identity.key) return;
const isStart = report.event === 'game_start';
const round = isStart ? 0 : report.round;
const newMatch = !match || match.map_key !== identity.key
|| (isStart && (match.status !== 'active' || match.last_event !== 'game_start'))
|| (!isStart && round < match.highest_round)
|| ((report.event === 'round_change' || report.event === 'heartbeat') && match.status !== 'active');
const newMatch =
!match ||
match.map_key !== identity.key ||
(isStart && (match.status !== 'active' || match.last_event !== 'game_start')) ||
(!isStart && round < match.highest_round) ||
((report.event === 'round_change' || report.event === 'heartbeat') &&
match.status !== 'active');
if (newMatch) {
if (match?.status === 'active') {
handle.prepare("UPDATE leaderboard_matches SET status = 'interrupted', ended_at = ? WHERE id = ?")
handle
.prepare(
"UPDATE leaderboard_matches SET status = 'interrupted', ended_at = ? WHERE id = ?"
)
.run(report.receivedAt, match.id);
}
const inserted = handle.prepare(`INSERT INTO leaderboard_matches
const inserted = handle
.prepare(
`INSERT INTO leaderboard_matches
(server, game, map, map_key, map_name, started_at, last_received_at, start_observed, last_event, last_fingerprint)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
.run(report.server, identity.game, identity.map, identity.key, identity.name,
report.receivedAt, report.receivedAt, Number(isStart), report.event, signature);
match = { id: Number(inserted.lastInsertRowid), game: identity.game, map_key: identity.key,
status: 'active', highest_round: 0, last_event: report.event, last_fingerprint: signature };
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(
report.server,
identity.game,
identity.map,
identity.key,
identity.name,
report.receivedAt,
report.receivedAt,
Number(isStart),
report.event,
signature
);
match = {
id: Number(inserted.lastInsertRowid),
game: identity.game,
map_key: identity.key,
status: 'active',
highest_round: 0,
last_event: report.event,
last_fingerprint: signature
};
}
if (!match) return;
if (isStart) {
handle.prepare('UPDATE leaderboard_matches SET start_players = COALESCE(start_players, ?) WHERE id = ?')
handle
.prepare(
'UPDATE leaderboard_matches SET start_players = COALESCE(start_players, ?) WHERE id = ?'
)
.run(report.players.length, match.id);
}
const previous = new Map((handle.prepare('SELECT guid, last_round, present FROM leaderboard_match_players WHERE match_id = ?')
.all(match.id) as Participation[]).map(player => [player.guid, player]));
handle.prepare('UPDATE leaderboard_match_players SET present = 0 WHERE match_id = ?').run(match.id);
const saveIdentity = handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
const previous = new Map(
(
handle
.prepare(
'SELECT guid, last_round, present FROM leaderboard_match_players WHERE match_id = ?'
)
.all(match.id) as Participation[]
).map((player) => [player.guid, player])
);
handle
.prepare('UPDATE leaderboard_match_players SET present = 0 WHERE match_id = ?')
.run(match.id);
const saveIdentity =
handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET
name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END,
last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`);
@@ -108,33 +161,76 @@ export function recordLeaderboardTelemetry(handle: Database.Database, report: Ga
const completedRounds = Math.max(0, round - 1);
for (const player of report.players) {
const prior = previous.get(player.guid);
const survived = !isStart && prior?.present ? Math.max(0, completedRounds - prior.last_round) : 0;
saveIdentity.run(identity.game, player.guid, player.name, report.receivedAt, report.receivedAt);
const survived =
!isStart && prior?.present ? Math.max(0, completedRounds - prior.last_round) : 0;
saveIdentity.run(
identity.game,
player.guid,
player.name,
report.receivedAt,
report.receivedAt
);
const stats = 'kills' in player ? player : null;
savePlayer.run(match.id, player.guid, round, survived,
stats?.kills ?? 0, stats?.headshots ?? 0, stats?.downs ?? 0, stats?.revives ?? 0, stats?.score ?? 0,
completedRounds, Number(!isStart), Number(report.event === 'game_end'), report.receivedAt, report.receivedAt, round, Number(isStart));
savePlayer.run(
match.id,
player.guid,
round,
survived,
stats?.kills ?? 0,
stats?.headshots ?? 0,
stats?.downs ?? 0,
stats?.revives ?? 0,
stats?.score ?? 0,
completedRounds,
Number(!isStart),
Number(report.event === 'game_end'),
report.receivedAt,
report.receivedAt,
round,
Number(isStart)
);
}
handle.prepare(`UPDATE leaderboard_matches SET highest_round = MAX(highest_round, ?),
handle
.prepare(
`UPDATE leaderboard_matches SET highest_round = MAX(highest_round, ?),
last_received_at = ?, last_event = ?, last_fingerprint = ?,
status = CASE WHEN ? = 'game_end' THEN 'completed' ELSE status END,
ended_at = CASE WHEN ? = 'game_end' THEN ? ELSE ended_at END WHERE id = ?`)
.run(round, report.receivedAt, report.event, signature, report.event, report.event, report.receivedAt, match.id);
ended_at = CASE WHEN ? = 'game_end' THEN ? ELSE ended_at END WHERE id = ?`
)
.run(
round,
report.receivedAt,
report.event,
signature,
report.event,
report.event,
report.receivedAt,
match.id
);
}
/** Seed the single surviving scoreboard per server once; older games are gone. */
export function backfillLeaderboard(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'leaderboard_v1'").get()) return;
handle.transaction(() => {
const reports = handle.prepare(`SELECT payload FROM server_telemetry WHERE payload IS NOT NULL ORDER BY received_at, server`)
const reports = handle
.prepare(
`SELECT payload FROM server_telemetry WHERE payload IS NOT NULL ORDER BY received_at, server`
)
.all() as Array<{ payload: string }>;
for (const row of reports) {
let report: GameTelemetry;
try {
const stored = JSON.parse(row.payload);
if (typeof stored.receivedAt !== 'string' || !Number.isFinite(Date.parse(stored.receivedAt))) continue;
if (
typeof stored.receivedAt !== 'string' ||
!Number.isFinite(Date.parse(stored.receivedAt))
)
continue;
report = { ...parseGameEvent(stored), receivedAt: stored.receivedAt };
} catch { continue; } // Legacy malformed snapshots cannot establish player records.
} catch {
continue;
} // Legacy malformed snapshots cannot establish player records.
recordLeaderboardTelemetry(handle, report);
}
handle.prepare("INSERT INTO settings (key, value) VALUES ('leaderboard_v1', '1')").run();
+39 -12
View File
@@ -1,5 +1,10 @@
import type Database from 'better-sqlite3';
import type { LeaderboardPlayerCount, PersonalBestMetric, PersonalBestRecords, RecordEvidence } from '../../leaderboard.js';
import type {
LeaderboardPlayerCount,
PersonalBestMetric,
PersonalBestRecords,
RecordEvidence
} from '../../leaderboard.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
interface RecordRequest extends Record<PersonalBestMetric, number> {
@@ -10,14 +15,27 @@ interface RecordRequest extends Record<PersonalBestMetric, number> {
}
/** Fetch evidence only for requested players and scopes, never rank the whole fleet's history. */
export function getPersonalBestRecords(handle: Database.Database, requests: RecordRequest[]): PersonalBestRecords[] {
export function getPersonalBestRecords(
handle: Database.Database,
requests: RecordRequest[]
): PersonalBestRecords[] {
const result: PersonalBestRecords[] = [];
// Page requests are at most 100 players; chunk unusually large profile map collections too.
for (let offset = 0; offset < requests.length; offset += 100) {
const batch = requests.slice(offset, offset + 100);
const parameters = batch.flatMap((request, index) => [index, request.guid, request.game ?? null,
request.map ?? null, request.players ?? null, request.highestRound, request.bestKills, request.bestScore]);
const rows = handle.prepare(`WITH requested (requestId, guid, game, map, players, highestRound, bestKills, bestScore) AS (
const parameters = batch.flatMap((request, index) => [
index,
request.guid,
request.game ?? null,
request.map ?? null,
request.players ?? null,
request.highestRound,
request.bestKills,
request.bestScore
]);
const rows = handle
.prepare(
`WITH requested (requestId, guid, game, map, players, highestRound, bestKills, bestScore) AS (
VALUES ${batch.map(() => '(?, ?, ?, ?, ?, ?, ?, ?)').join(', ')}
), candidates AS (
SELECT r.requestId, m.id AS matchId, m.game, m.map_key AS mapKey, m.map_name AS mapName,
@@ -41,14 +59,23 @@ export function getPersonalBestRecords(handle: Database.Database, requests: Reco
SELECT *, ROW_NUMBER() OVER (PARTITION BY requestId, metric ORDER BY completionOrder, matchId) AS choice
FROM valuesByMetric
) SELECT requestId, metric, value, matchId, game, mapKey, mapName, server, startedAt, endedAt,
startObserved, firstObservedRound, presentAtStart FROM ranked WHERE choice = 1`)
.all(...parameters) as Array<Omit<RecordEvidence, 'startObserved' | 'presentAtStart'> & {
requestId: number; metric: PersonalBestMetric; startObserved: number; presentAtStart: number | null;
}>;
const records = batch.map(() => ({} as PersonalBestRecords));
startObserved, firstObservedRound, presentAtStart FROM ranked WHERE choice = 1`
)
.all(...parameters) as Array<
Omit<RecordEvidence, 'startObserved' | 'presentAtStart'> & {
requestId: number;
metric: PersonalBestMetric;
startObserved: number;
presentAtStart: number | null;
}
>;
const records = batch.map(() => ({}) as PersonalBestRecords);
for (const { requestId, metric, startObserved, presentAtStart, ...record } of rows) {
records[requestId][metric] = { ...record, startObserved: Boolean(startObserved),
presentAtStart: presentAtStart === null ? null : Boolean(presentAtStart) };
records[requestId][metric] = {
...record,
startObserved: Boolean(startObserved),
presentAtStart: presentAtStart === null ? null : Boolean(presentAtStart)
};
}
result.push(...records);
}
+166 -54
View File
@@ -2,19 +2,34 @@ import { db } from '../db/index.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
import { getPersonalBestRecords } from './leaderboard-records.js';
import { countsAsGamePlayed } from './leaderboard-participation.js';
import { EFFICIENCY_MIN_GAMES, EFFICIENCY_MIN_KILLS, EFFICIENCY_MIN_DOWNS,
isEfficiencyMetric } from '../../leaderboard-efficiency.js';
import { leaderboardMetrics, type LeaderboardEntry, type LeaderboardMap, type LeaderboardMetric,
type LeaderboardOptions, type LeaderboardResponse } from '../../leaderboard.js';
import {
EFFICIENCY_MIN_GAMES,
EFFICIENCY_MIN_KILLS,
EFFICIENCY_MIN_DOWNS,
isEfficiencyMetric
} from '../../leaderboard-efficiency.js';
import {
leaderboardMetrics,
type LeaderboardEntry,
type LeaderboardMap,
type LeaderboardMetric,
type LeaderboardOptions,
type LeaderboardResponse
} from '../../leaderboard.js';
const metrics = new Set<string>(leaderboardMetrics.map(metric => metric.key));
const metrics = new Set<string>(leaderboardMetrics.map((metric) => metric.key));
export const MAX_LEADERBOARD_OFFSET = 1_000_000;
type RankedEntry = Omit<LeaderboardEntry, 'records'>;
interface RankedScope { revision: number; entries: RankedEntry[]; maps: LeaderboardMap[]; summary: LeaderboardResponse['summary'] }
interface RankedScope {
revision: number;
entries: RankedEntry[];
maps: LeaderboardMap[];
summary: LeaderboardResponse['summary'];
}
const scopeCaches = new WeakMap<object, Map<string, RankedScope>>();
// Match SQLite's built-in lower(), which folds ASCII rather than Unicode.
const searchText = (value: string) => value.replace(/[A-Z]/g, ch => ch.toLowerCase());
const searchText = (value: string) => value.replace(/[A-Z]/g, (ch) => ch.toLowerCase());
interface LeaderboardRow extends Omit<RankedEntry, 'guid'> {
// A left join keeps metadata available when the requested page has no players.
@@ -39,35 +54,78 @@ export function getLeaderboard(options: LeaderboardOptions = {}): LeaderboardRes
}
/** Select the exact player after ranking everyone eligible in the requested scope. */
export function getPlayerLeaderboardEntry(guid: string,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players' | 'metric'> = {}): LeaderboardEntry | null {
return readLeaderboard({ game: options.game, map: options.map, players: options.players, metric: options.metric, limit: 1 }, guid);
export function getPlayerLeaderboardEntry(
guid: string,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players' | 'metric'> = {}
): LeaderboardEntry | null {
return readLeaderboard(
{
game: options.game,
map: options.map,
players: options.players,
metric: options.metric,
limit: 1
},
guid
);
}
function addRecordSources(handle: ReturnType<typeof db>, entries: RankedEntry[], options: LeaderboardOptions): LeaderboardEntry[] {
const records = getPersonalBestRecords(handle, entries.map(entry => ({
...entry, game: options.game || undefined, map: options.map || undefined, players: options.players
})));
function addRecordSources(
handle: ReturnType<typeof db>,
entries: RankedEntry[],
options: LeaderboardOptions
): LeaderboardEntry[] {
const records = getPersonalBestRecords(
handle,
entries.map((entry) => ({
...entry,
game: options.game || undefined,
map: options.map || undefined,
players: options.players
}))
);
return entries.map((entry, index) => ({ ...entry, records: records[index] }));
}
function readLeaderboard(options: LeaderboardOptions): LeaderboardResponse;
function readLeaderboard(options: LeaderboardOptions, exactGuid: string): LeaderboardEntry | null;
function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): LeaderboardResponse | LeaderboardEntry | null {
function readLeaderboard(
options: LeaderboardOptions,
exactGuid?: string
): LeaderboardResponse | LeaderboardEntry | null {
const handle = db();
const metric = options.metric && isLeaderboardMetric(options.metric) ? options.metric : 'highestRound';
const limit = Number.isSafeInteger(options.limit) ? Math.min(100, Math.max(1, options.limit!)) : 50;
const offset = Number.isSafeInteger(options.offset) ? Math.min(MAX_LEADERBOARD_OFFSET, Math.max(0, options.offset!)) : 0;
const conditions = ["m.game IN ('t5', 't6')", "m.status = 'completed'", 'mp.has_stats = 1', `m.highest_round >= ${MIN_RECORDED_ROUND}`,
`mp.highest_round >= ${MIN_RECORDED_ROUND}`];
const metric =
options.metric && isLeaderboardMetric(options.metric) ? options.metric : 'highestRound';
const limit = Number.isSafeInteger(options.limit)
? Math.min(100, Math.max(1, options.limit!))
: 50;
const offset = Number.isSafeInteger(options.offset)
? Math.min(MAX_LEADERBOARD_OFFSET, Math.max(0, options.offset!))
: 0;
const conditions = [
"m.game IN ('t5', 't6')",
"m.status = 'completed'",
'mp.has_stats = 1',
`m.highest_round >= ${MIN_RECORDED_ROUND}`,
`mp.highest_round >= ${MIN_RECORDED_ROUND}`
];
const parameters: Array<string | number> = [];
if (options.game) { conditions.push('m.game = ?'); parameters.push(options.game); }
if (options.map) { conditions.push('m.map_key = ?'); parameters.push(options.map); }
if (options.players) { conditions.push('m.start_players = ?'); parameters.push(options.players); }
if (options.game) {
conditions.push('m.game = ?');
parameters.push(options.game);
}
if (options.map) {
conditions.push('m.map_key = ?');
parameters.push(options.map);
}
if (options.players) {
conditions.push('m.start_players = ?');
parameters.push(options.players);
}
const qualification = isEfficiencyMetric(metric)
? `WHERE measures.gamesPlayed >= ${EFFICIENCY_MIN_GAMES} AND measures.${metric} IS NOT NULL`
+ (metric === 'headshotRate' ? ` AND measures.kills >= ${EFFICIENCY_MIN_KILLS}` : '')
+ (metric === 'killsPerDown' ? ` AND measures.downs >= ${EFFICIENCY_MIN_DOWNS}` : '')
? `WHERE measures.gamesPlayed >= ${EFFICIENCY_MIN_GAMES} AND measures.${metric} IS NOT NULL` +
(metric === 'headshotRate' ? ` AND measures.kills >= ${EFFICIENCY_MIN_KILLS}` : '') +
(metric === 'killsPerDown' ? ` AND measures.downs >= ${EFFICIENCY_MIN_DOWNS}` : '')
: '';
// A board reuses observations for player totals and distinct match metadata.
// Exact-player requests only consume the totals, so they can stream the join once.
@@ -100,14 +158,31 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
FROM measures JOIN identities p ON p.guid = measures.guid AND p.recency = 1 ${qualification}
)`;
let revision = -1;
try { revision = (handle.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'").get() as { revision: number }).revision; }
catch { /* Standalone history tools can operate without cache metadata. */ }
try {
revision = (
handle
.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'")
.get() as { revision: number }
).revision;
} catch {
/* Standalone history tools can operate without cache metadata. */
}
let cache = scopeCaches.get(handle);
if (!cache) { cache = new Map(); scopeCaches.set(handle, cache); }
const cacheKey = JSON.stringify([options.game ?? '', options.map ?? '', options.players ?? '', metric]);
if (!cache) {
cache = new Map();
scopeCaches.set(handle, cache);
}
const cacheKey = JSON.stringify([
options.game ?? '',
options.map ?? '',
options.players ?? '',
metric
]);
let saved = cache.get(cacheKey);
if (!saved || revision < 0 || saved.revision !== revision) {
const rows = handle.prepare(`${aggregated}, playerSummary AS (
const rows = handle
.prepare(
`${aggregated}, playerSummary AS (
SELECT COUNT(*) AS players, COALESCE(SUM(kills), 0) AS kills,
COALESCE(SUM(roundsSurvived), 0) AS roundsSurvived,
COALESCE(MAX(highestRound), 0) AS highestRound FROM totals
@@ -121,36 +196,73 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
metadata.roundsSurvived AS summaryRounds, metadata.highestRound AS summaryHighestRound,
metadata.firstTrackedAt AS summaryFirstTrackedAt, metadata.updatedAt AS summaryUpdatedAt
FROM metadata LEFT JOIN ranked ON 1
ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`)
.all(...parameters) as LeaderboardRow[];
const rankedEntries = rows.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
.map(({ matchingTotal, summaryPlayers, summaryMatches, summaryKills, summaryRounds,
summaryHighestRound, summaryFirstTrackedAt, summaryUpdatedAt, ...entry }) => entry);
const metadata = rows[0];
const summary: LeaderboardResponse['summary'] = {
players: metadata.summaryPlayers, matches: metadata.summaryMatches, kills: metadata.summaryKills,
roundsSurvived: metadata.summaryRounds, highestRound: metadata.summaryHighestRound,
firstTrackedAt: metadata.summaryFirstTrackedAt, updatedAt: metadata.summaryUpdatedAt
};
// Keep qualifying map variants available, even after their server is removed.
const maps = handle.prepare(`SELECT map_key AS key, game, map, map_name AS name
ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`
)
.all(...parameters) as LeaderboardRow[];
const rankedEntries = rows
.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
.map(
({
matchingTotal,
summaryPlayers,
summaryMatches,
summaryKills,
summaryRounds,
summaryHighestRound,
summaryFirstTrackedAt,
summaryUpdatedAt,
...entry
}) => entry
);
const metadata = rows[0];
const summary: LeaderboardResponse['summary'] = {
players: metadata.summaryPlayers,
matches: metadata.summaryMatches,
kills: metadata.summaryKills,
roundsSurvived: metadata.summaryRounds,
highestRound: metadata.summaryHighestRound,
firstTrackedAt: metadata.summaryFirstTrackedAt,
updatedAt: metadata.summaryUpdatedAt
};
// Keep qualifying map variants available, even after their server is removed.
const maps = handle
.prepare(
`SELECT map_key AS key, game, map, map_name AS name
FROM leaderboard_matches WHERE game IN ('t5', 't6') AND status = 'completed'
AND highest_round >= ${MIN_RECORDED_ROUND}${options.game ? ' AND game = ?' : ''}
GROUP BY game, map_key HAVING id = MAX(id)
ORDER BY game, name COLLATE NOCASE, map_key`)
.all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
saved = { revision, entries: rankedEntries, maps, summary };
cache.delete(cacheKey);
if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
// Bound total retained players as well as the number of scopes.
while (cache.size > 8 || [...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000) cache.delete(cache.keys().next().value!);
ORDER BY game, name COLLATE NOCASE, map_key`
)
.all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
saved = { revision, entries: rankedEntries, maps, summary };
cache.delete(cacheKey);
if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
// Bound total retained players as well as the number of scopes.
while (
cache.size > 8 ||
[...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000
)
cache.delete(cache.keys().next().value!);
}
if (exactGuid !== undefined) {
const entry = saved.entries.find(entry => entry.guid === exactGuid);
const entry = saved.entries.find((entry) => entry.guid === exactGuid);
return entry ? addRecordSources(handle, [entry], options)[0] : null;
}
const query = searchText(options.q?.trim() ?? '');
const matching = query ? saved.entries.filter(entry => searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)) : saved.entries;
const matching = query
? saved.entries.filter(
(entry) =>
searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)
)
: saved.entries;
const entries = addRecordSources(handle, matching.slice(offset, offset + limit), options);
return { entries, maps: saved.maps, total: matching.length, limit, offset, metric, summary: saved.summary };
return {
entries,
maps: saved.maps,
total: matching.length,
limit,
offset,
metric,
summary: saved.summary
};
}
+64 -17
View File
@@ -6,7 +6,9 @@ export function scriptCatalog(kind?: ArtifactKind) {
const version = (alias: string) => `CASE WHEN ${alias}.id IS NOT NULL THEN json_object(
'id',${alias}.id,'version',${alias}.version,'release_version',${alias}.release_version,
'sha256',${alias}.sha256,'size',${alias}.size,'state',${alias}.state,'uploaded_at',${alias}.uploaded_at) END`;
const rows = db().prepare(`SELECT s.*, ${version('c')} current_json, ${version('v')} latest_json,
const rows = db()
.prepare(
`SELECT s.*, ${version('c')} current_json, ${version('v')} latest_json,
COALESCE(counts.n,0) versionCount, active.beta_id runningBetaId
FROM scripts s
LEFT JOIN (SELECT script_id, COUNT(*) n, MAX(version) latest FROM script_versions GROUP BY script_id) counts ON counts.script_id=s.id
@@ -14,32 +16,77 @@ export function scriptCatalog(kind?: ArtifactKind) {
LEFT JOIN script_versions c ON c.id=s.current_version_id
LEFT JOIN (SELECT bs.script_id, MIN(b.id) beta_id FROM beta_scripts bs JOIN betas b ON b.id=bs.beta_id
WHERE b.state IN ('running','ready-promote') GROUP BY bs.script_id) active ON active.script_id=s.id
${kind ? 'WHERE s.kind=?' : ''} ORDER BY s.name`).all(...(kind ? [kind] : [])) as Array<ScriptRow & {
current_json: string | null; latest_json: string | null; versionCount: number; runningBetaId: number | null;
}>;
const servers = db().prepare('SELECT name, game, map FROM servers ORDER BY name').all() as { name: string; game: string | null; map: string | null }[];
return rows.map(({ current_json, latest_json, ...row }) => ({ ...row,
currentVersion: current_json ? JSON.parse(current_json) as ScriptVersionRow : null,
latestVersion: latest_json ? JSON.parse(latest_json) as ScriptVersionRow : null,
servers: row.scope_type && row.scope_value ? servers.filter(server => server[row.scope_type!] === row.scope_value).map(server => server.name) : []
${kind ? 'WHERE s.kind=?' : ''} ORDER BY s.name`
)
.all(...(kind ? [kind] : [])) as Array<
ScriptRow & {
current_json: string | null;
latest_json: string | null;
versionCount: number;
runningBetaId: number | null;
}
>;
const servers = db().prepare('SELECT name, game, map FROM servers ORDER BY name').all() as {
name: string;
game: string | null;
map: string | null;
}[];
return rows.map(({ current_json, latest_json, ...row }) => ({
...row,
currentVersion: current_json ? (JSON.parse(current_json) as ScriptVersionRow) : null,
latestVersion: latest_json ? (JSON.parse(latest_json) as ScriptVersionRow) : null,
servers:
row.scope_type && row.scope_value
? servers
.filter((server) => server[row.scope_type!] === row.scope_value)
.map((server) => server.name)
: []
}));
}
export function betaScriptMetadata(ids: number[]) {
const rows = db().prepare(`SELECT bs.beta_id, bs.script_id id, COALESCE(s.name,'(deleted)') name,
const rows = db()
.prepare(
`SELECT bs.beta_id, bs.script_id id, COALESCE(s.name,'(deleted)') name,
COALESCE(s.target_rel_path,'') target_rel_path, COALESCE(s.kind,bs.kind,'script') kind,
v.id version_id, v.version, v.release_version, v.sha256
FROM beta_scripts bs LEFT JOIN scripts s ON s.id=bs.script_id
LEFT JOIN script_versions v ON v.id=bs.script_version_id
WHERE bs.beta_id IN (SELECT value FROM json_each(?)) ORDER BY bs.script_id`).all(JSON.stringify(ids)) as Array<{
beta_id: number; id: number; name: string; target_rel_path: string; kind: ArtifactKind;
version_id: number | null; version: number; release_version: string | null; sha256: string;
}>;
const grouped = new Map<number, Array<{ id: number; name: string; target_rel_path: string; kind: ArtifactKind;
version: { id: number; version: number; release_version: string | null; sha256: string } | null }>>();
WHERE bs.beta_id IN (SELECT value FROM json_each(?)) ORDER BY bs.script_id`
)
.all(JSON.stringify(ids)) as Array<{
beta_id: number;
id: number;
name: string;
target_rel_path: string;
kind: ArtifactKind;
version_id: number | null;
version: number;
release_version: string | null;
sha256: string;
}>;
const grouped = new Map<
number,
Array<{
id: number;
name: string;
target_rel_path: string;
kind: ArtifactKind;
version: {
id: number;
version: number;
release_version: string | null;
sha256: string;
} | null;
}>
>();
for (const { beta_id, version_id, version, release_version, sha256, ...script } of rows) {
const group = grouped.get(beta_id) ?? [];
group.push({ ...script, version: version_id === null ? null : { id: version_id, version, release_version, sha256 } });
group.push({
...script,
version:
version_id === null ? null : { id: version_id, version, release_version, sha256 }
});
grouped.set(beta_id, group);
}
return grouped;
+15 -6
View File
@@ -2,18 +2,27 @@ import { db } from '../db/index.js';
/** Reports from before the latest lifecycle submission describe the old process. */
export function latestGameSubmission(server: string): string | null {
return (db().prepare(`SELECT MAX(at) AS at FROM (
return (
db()
.prepare(
`SELECT MAX(at) AS at FROM (
SELECT MAX(submitted_at) AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`
)
.get(server, server) as { at: string | null }
).at;
}
/** Look up each registered server through the covering submission index. */
export function latestGameSubmissions(): Map<string, string | null> {
const rows = db().prepare(`SELECT server, MAX(at) AS at FROM (
const rows = db()
.prepare(
`SELECT server, MAX(at) AS at FROM (
SELECT s.name AS server, (SELECT MAX(submitted_at) FROM server_deployments
WHERE server=s.name AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')) AS at
FROM servers s
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as
{ server: string; at: string | null }[];
return new Map(rows.map(row => [row.server, row.at]));
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`
)
.all() as { server: string; at: string | null }[];
return new Map(rows.map((row) => [row.server, row.at]));
}
+60 -15
View File
@@ -14,7 +14,8 @@ export const KNOWN_LOGS = {
} as const;
export type LogKey = keyof typeof KNOWN_LOGS;
export const gameLogKey = (server: string): LogKey => gameOfServer(server) === 't6' ? 'games_zm' : 'games_sp';
export const gameLogKey = (server: string): LogKey =>
gameOfServer(server) === 't6' ? 'games_zm' : 'games_sp';
export function isLogKey(v: string): v is LogKey {
return Object.prototype.hasOwnProperty.call(KNOWN_LOGS, v);
@@ -45,33 +46,62 @@ export function resolveLogPath(serverName: string, key: LogKey): string | null {
export interface TailResult {
lines: string[];
size: number;
mtime: string; // ISO
mtime: string; // ISO
truncated: boolean; // true when we didn't read from the start of the file
}
export interface LogCursor {
fileId: string; offset: number; size: number; modified: string;
anchorLength: number; anchorHash: string;
fileId: string;
offset: number;
size: number;
modified: string;
anchorLength: number;
anchorHash: string;
}
export interface LogChunk {
contentBase64: string; size: number; mtime: string; truncated: boolean;
cursor: LogCursor; reset: boolean;
contentBase64: string;
size: number;
mtime: string;
truncated: boolean;
cursor: LogCursor;
reset: boolean;
}
/** Bounded byte reads; an anchor before the offset detects copy-truncate and regrowth. */
export async function readLogChunk(path: string, cursor?: LogCursor): Promise<LogChunk | TailResult | null> {
export async function readLogChunk(
path: string,
cursor?: LogCursor
): Promise<LogChunk | TailResult | null> {
const remote = await nativeLogChunk(path, cursor);
if (remote !== undefined) return remote;
let file;
try { file = await open(path, 'r'); }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; throw error; }
try {
file = await open(path, 'r');
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw error;
}
try {
const st = await file.stat();
if (!st.isFile()) return null;
const fileId = `${st.dev}:${st.ino}`, modified = `${st.mtimeMs}:${st.ctimeMs}`, size = st.size;
const fileId = `${st.dev}:${st.ino}`,
modified = `${st.mtimeMs}:${st.ctimeMs}`,
size = st.size;
const mtime = st.mtime.toISOString();
if (cursor?.fileId === fileId && cursor.modified === modified && cursor.size === size && cursor.offset === size) {
return { contentBase64: '', size, mtime, truncated: cursor.offset > 0, cursor, reset: false };
if (
cursor?.fileId === fileId &&
cursor.modified === modified &&
cursor.size === size &&
cursor.offset === size
) {
return {
contentBase64: '',
size,
mtime,
truncated: cursor.offset > 0,
cursor,
reset: false
};
}
const hash = (value: Buffer) => createHash('sha256').update(value).digest('hex');
let reset = !cursor || cursor.fileId !== fileId || cursor.offset > size;
@@ -86,9 +116,24 @@ export async function readLogChunk(path: string, cursor?: LogCursor): Promise<Lo
const offset = start + read.bytesRead;
const anchor = Buffer.alloc(Math.min(256, offset));
await file.read(anchor, 0, anchor.length, offset - anchor.length);
return { contentBase64: content.subarray(0, read.bytesRead).toString('base64'), size, mtime, truncated: start > 0, reset,
cursor: { fileId, modified, size, offset, anchorLength: anchor.length, anchorHash: hash(anchor) } };
} finally { await file.close(); }
return {
contentBase64: content.subarray(0, read.bytesRead).toString('base64'),
size,
mtime,
truncated: start > 0,
reset,
cursor: {
fileId,
modified,
size,
offset,
anchorLength: anchor.length,
anchorHash: hash(anchor)
}
};
} finally {
await file.close();
}
}
const DEFAULT_TAIL_LINES = 200;
+92 -27
View File
@@ -1,6 +1,12 @@
import { db } from '../db/index.js';
import { matchChatTimeline } from './chat.js';
import { MIN_RECORDED_ROUND, type MatchDetail, type MatchPlayer, type MatchHistoryResponse, type ServerMatch } from '../../match-history.js';
import {
MIN_RECORDED_ROUND,
type MatchDetail,
type MatchPlayer,
type MatchHistoryResponse,
type ServerMatch
} from '../../match-history.js';
import { readMatchQuery, type MatchHistoryQuery } from '../../match-history-query.js';
@@ -15,29 +21,69 @@ const matchSummary = `m.id, m.game, m.highest_round AS observedRound, m.server,
MAX(0, ROUND((julianday(m.ended_at) - julianday(m.started_at)) * 86400000)) END AS durationMs,
CASE WHEN m.status = 'completed' THEN m.highest_round END AS finalRound`;
export function serverMatchHistory(server?: string, limit = 25, query: MatchHistoryQuery = readMatchQuery(new URLSearchParams()), now = Date.now()): MatchHistoryResponse {
export function serverMatchHistory(
server?: string,
limit = 25,
query: MatchHistoryQuery = readMatchQuery(new URLSearchParams()),
now = Date.now()
): MatchHistoryResponse {
const handle = db();
const clauses = [`m.highest_round >= ${MIN_RECORDED_ROUND}`];
const parameters: Array<string | number> = [];
if (server !== undefined) { clauses.push('m.server = ?'); parameters.push(server); }
if (query.game) { clauses.push('m.game = ?'); parameters.push(query.game); }
const maps = handle.prepare(`SELECT m.map_key AS key, MAX(m.map_name) AS name, m.game
FROM leaderboard_matches m WHERE ${clauses.join(' AND ')} GROUP BY m.map_key, m.game ORDER BY name COLLATE NOCASE, game`)
if (server !== undefined) {
clauses.push('m.server = ?');
parameters.push(server);
}
if (query.game) {
clauses.push('m.game = ?');
parameters.push(query.game);
}
const maps = handle
.prepare(
`SELECT m.map_key AS key, MAX(m.map_name) AS name, m.game
FROM leaderboard_matches m WHERE ${clauses.join(' AND ')} GROUP BY m.map_key, m.game ORDER BY name COLLATE NOCASE, game`
)
.all(...parameters) as MatchHistoryResponse['maps'];
if (query.map) { clauses.push('m.map_key = ?'); parameters.push(query.map); }
if (query.status) { clauses.push('m.status = ?'); parameters.push(query.status); }
if (query.days) { clauses.push('julianday(m.started_at) >= julianday(?)'); parameters.push(new Date(now - query.days * 86400000).toISOString()); }
if (query.map) {
clauses.push('m.map_key = ?');
parameters.push(query.map);
}
if (query.status) {
clauses.push('m.status = ?');
parameters.push(query.status);
}
if (query.days) {
clauses.push('julianday(m.started_at) >= julianday(?)');
parameters.push(new Date(now - query.days * 86400000).toISOString());
}
// Search readable instance names as well as their stored identifiers. Tokens allow "Town Lobby 1".
for (const token of query.q.toLowerCase().replace(/\blobby\s+(?=\d+\b)/g, '').split(/[\s_]+/).filter(Boolean)) {
clauses.push("instr(lower(m.server || ' ' || m.map_name), ?) > 0"); parameters.push(token);
for (const token of query.q
.toLowerCase()
.replace(/\blobby\s+(?=\d+\b)/g, '')
.split(/[\s_]+/)
.filter(Boolean)) {
clauses.push("instr(lower(m.server || ' ' || m.map_name), ?) > 0");
parameters.push(token);
}
const where = `WHERE ${clauses.join(' AND ')}`;
const total = (handle.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`).get(...parameters) as { total: number }).total;
const total = (
handle
.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`)
.get(...parameters) as { total: number }
).total;
const page = Math.min(query.page, Math.max(1, Math.ceil(total / limit)));
const column = { started: 'startedAt', round: 'observedRound', duration: 'durationMs', players: 'uniquePlayers' }[query.sort];
const column = {
started: 'startedAt',
round: 'observedRound',
duration: 'durationMs',
players: 'uniquePlayers'
}[query.sort];
const direction = query.direction === 'asc' ? 'ASC' : 'DESC';
const matches = handle.prepare(`SELECT ${matchSummary} FROM leaderboard_matches m
${where} ORDER BY ${column} IS NULL ASC, ${column} ${direction}, m.id DESC LIMIT ? OFFSET ?`)
const matches = handle
.prepare(
`SELECT ${matchSummary} FROM leaderboard_matches m
${where} ORDER BY ${column} IS NULL ASC, ${column} ${direction}, m.id DESC LIMIT ? OFFSET ?`
)
.all(...parameters, limit, (page - 1) * limit) as ServerMatch[];
return { matches, total, page, limit, maps };
}
@@ -46,13 +92,20 @@ export function serverMatchHistory(server?: string, limit = 25, query: MatchHist
export function getMatchDetail(id: number): MatchDetail | null {
if (!Number.isSafeInteger(id) || id < 1) return null;
const handle = db();
const match = handle.prepare(`SELECT ${matchSummary}, m.start_observed AS startObserved
const match = handle
.prepare(
`SELECT ${matchSummary}, m.start_observed AS startObserved
FROM leaderboard_matches m WHERE m.id = ? AND m.status = 'completed'
AND m.highest_round >= ${MIN_RECORDED_ROUND}`)
.get(id) as (Omit<MatchDetail, 'players' | 'chat' | 'startObserved'> & { startObserved: number }) | undefined;
AND m.highest_round >= ${MIN_RECORDED_ROUND}`
)
.get(id) as
| (Omit<MatchDetail, 'players' | 'chat' | 'startObserved'> & { startObserved: number })
| undefined;
if (!match) return null;
const players = handle.prepare(`SELECT p.guid,
const players = handle
.prepare(
`SELECT p.guid,
COALESCE((SELECT identity.name FROM leaderboard_players identity
WHERE identity.guid = p.guid AND identity.game IN ('t5', 't6')
ORDER BY identity.last_seen_at DESC, identity.game LIMIT 1),
@@ -76,12 +129,24 @@ export function getMatchDetail(id: number): MatchDetail | null {
AND history.highest_round >= ${MIN_RECORDED_ROUND}
AND recorded.highest_round >= ${MIN_RECORDED_ROUND})) AS canViewProfile
FROM leaderboard_match_players p WHERE p.match_id = ?
ORDER BY hasStats DESC, kills DESC, highestRound DESC, name COLLATE NOCASE, p.guid`)
.all(match.game, id) as Array<Omit<MatchPlayer, 'hasStats' | 'presentAtEnd' | 'canViewProfile'> & {
hasStats: number; presentAtEnd: number; canViewProfile: number;
}>;
return { ...match, chat: matchChatTimeline(id), startObserved: Boolean(match.startObserved), players: players.map(player => ({
...player, hasStats: Boolean(player.hasStats), presentAtEnd: Boolean(player.presentAtEnd),
canViewProfile: Boolean(player.canViewProfile)
})) };
ORDER BY hasStats DESC, kills DESC, highestRound DESC, name COLLATE NOCASE, p.guid`
)
.all(match.game, id) as Array<
Omit<MatchPlayer, 'hasStats' | 'presentAtEnd' | 'canViewProfile'> & {
hasStats: number;
presentAtEnd: number;
canViewProfile: number;
}
>;
return {
...match,
chat: matchChatTimeline(id),
startObserved: Boolean(match.startObserved),
players: players.map((player) => ({
...player,
hasStats: Boolean(player.hasStats),
presentAtEnd: Boolean(player.presentAtEnd),
canViewProfile: Boolean(player.canViewProfile)
}))
};
}
+196 -49
View File
@@ -1,8 +1,18 @@
import { fleetSettings } from '../settings.js';
import { db } from '../db/index.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { parseMetricsQuery, type HeatmapCell, type MapMetrics, type MetricPoint, type MetricsData,
type MetricsDays, type MetricsQuery, type PreviousMetrics, type ServerMetrics, type Utilization } from '../../metrics.js';
import {
parseMetricsQuery,
type HeatmapCell,
type MapMetrics,
type MetricPoint,
type MetricsData,
type MetricsDays,
type MetricsQuery,
type PreviousMetrics,
type ServerMetrics,
type Utilization
} from '../../metrics.js';
export type { MapMetrics, Utilization } from '../../metrics.js';
@@ -17,10 +27,13 @@ export interface MetricSample {
export function recordMetrics(samples: MetricSample[], at = Date.now()): void {
const handle = db();
handle.transaction(() => {
const insert = handle.prepare('INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)');
const insert = handle.prepare(
'INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)'
);
for (const s of samples) insert.run(at, s.server, s.game, s.map, s.players, s.capacity);
const days = fleetSettings().metricsKeepDays;
if (days) handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - days * 86400_000);
if (days)
handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - days * 86400_000);
})();
}
@@ -42,52 +55,120 @@ function addObservation(total: Utilization, players: number | null, hours: numbe
* capped at the freshness limit; longer gaps contribute no observed or unknown time.
* Read the indexed history once and share the intervals across every aggregation. */
export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()): MetricsData {
const query = parseMetricsQuery(new URLSearchParams(typeof input === 'number' ? { days: String(input) }
: { days: String(input.days), game: input.game ?? '', map: input.map ?? '', timeZone: input.timeZone }));
const query = parseMetricsQuery(
new URLSearchParams(
typeof input === 'number'
? { days: String(input) }
: {
days: String(input.days),
game: input.game ?? '',
map: input.map ?? '',
timeZone: input.timeZone
}
)
);
const { days, ...scope } = query;
const since = now - days * 86400_000;
const bucketMs = days === 1 ? 3600_000 : 86400_000;
const handle = db();
// These small grouped results keep options and retained evidence independent
// of the chosen scope/window without transferring a second raw history scan.
const groups = handle.prepare(`SELECT game, map, MIN(at) AS first, MAX(at) AS latest
FROM metric_samples WHERE at <= ? GROUP BY game, map ORDER BY game, map`).all(now) as
{ game: string; map: string; first: number; latest: number }[];
const groups = handle
.prepare(
`SELECT game, map, MIN(at) AS first, MAX(at) AS latest
FROM metric_samples WHERE at <= ? GROUP BY game, map ORDER BY game, map`
)
.all(now) as { game: string; map: string; first: number; latest: number }[];
const matchesScope = (sample: { game: string; map: string }) =>
(scope.game === null || sample.game === scope.game) && (scope.map === null || sample.map === scope.map);
const evidenceBounds = (values: typeof groups) => values.reduce<{ first: number | null; latest: number | null }>((bounds, group) => ({
first: bounds.first === null ? group.first : Math.min(bounds.first, group.first),
latest: bounds.latest === null ? group.latest : Math.max(bounds.latest, group.latest)
}), { first: null, latest: null });
(scope.game === null || sample.game === scope.game) &&
(scope.map === null || sample.map === scope.map);
const evidenceBounds = (values: typeof groups) =>
values.reduce<{ first: number | null; latest: number | null }>(
(bounds, group) => ({
first: bounds.first === null ? group.first : Math.min(bounds.first, group.first),
latest:
bounds.latest === null ? group.latest : Math.max(bounds.latest, group.latest)
}),
{ first: null, latest: null }
);
const globalBounds = evidenceBounds(groups);
const scopedBounds = evidenceBounds(groups.filter(matchesScope));
const registered = new Set((handle.prepare('SELECT name FROM servers WHERE destroyed_at IS NULL').all() as { name: string }[])
.map(server => server.name));
const metadata = { firstRetainedSampleAt: globalBounds.first, latestSampleAt: globalBounds.latest,
registeredServerCount: registered.size, scopeFirstSampleAt: scopedBounds.first, scopeLatestSampleAt: scopedBounds.latest };
const scopeOptions = { games: [...new Set(groups.map(group => group.game))], maps: groups.map(({ game, map }) => ({ game, map })) };
const registered = new Set(
(
handle.prepare('SELECT name FROM servers WHERE destroyed_at IS NULL').all() as {
name: string;
}[]
).map((server) => server.name)
);
const metadata = {
firstRetainedSampleAt: globalBounds.first,
latestSampleAt: globalBounds.latest,
registeredServerCount: registered.size,
scopeFirstSampleAt: scopedBounds.first,
scopeLatestSampleAt: scopedBounds.latest
};
const scopeOptions = {
games: [...new Set(groups.map((group) => group.game))],
maps: groups.map(({ game, map }) => ({ game, map }))
};
const summary = emptyUtilization();
const previous: PreviousMetrics = { since: since - days * 86400_000, until: since, summary: emptyUtilization(),
firstSampleAt: null, lastSampleAt: null, partialHistory: false };
const previous: PreviousMetrics = {
since: since - days * 86400_000,
until: since,
summary: emptyUtilization(),
firstSampleAt: null,
lastSampleAt: null,
partialHistory: false
};
const serverTotals = new Map<string, ServerMetrics>();
const mapTotals = new Map<string, Map<string, MapMetrics>>();
const points: MetricPoint[] = [];
for (let at = Math.floor(since / bucketMs) * bucketMs; at < now; at += bucketMs) {
const end = Math.min(at + bucketMs, now);
const start = Math.max(at, since);
points.push({ at, startAt: start, endAt: end, partial: start !== at || end !== at + bucketMs, ...emptyUtilization() });
points.push({
at,
startAt: start,
endAt: end,
partial: start !== at || end !== at + bucketMs,
...emptyUtilization()
});
}
const cells: HeatmapCell[] = Array.from({ length: 7 * 24 }, (_, index) => ({
weekday: Math.floor(index / 24), hour: index % 24, distinctDates: 0, ...emptyUtilization()
weekday: Math.floor(index / 24),
hour: index % 24,
distinctDates: 0,
...emptyUtilization()
}));
const cellDates = cells.map(() => new Set<string>());
const calendar = new Intl.DateTimeFormat('en-US', { timeZone: scope.timeZone,
year: 'numeric', month: '2-digit', day: '2-digit', weekday: 'short', hour: 'numeric', hourCycle: 'h23' });
const weekdays: Record<string, number> = { Mon: 0, Tue: 1, Wed: 2, Thu: 3, Fri: 4, Sat: 5, Sun: 6 };
const calendar = new Intl.DateTimeFormat('en-US', {
timeZone: scope.timeZone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
weekday: 'short',
hour: 'numeric',
hourCycle: 'h23'
});
const weekdays: Record<string, number> = {
Mon: 0,
Tue: 1,
Wed: 2,
Thu: 3,
Fri: 4,
Sat: 5,
Sun: 6
};
let cachedMinute = NaN;
let localCell = { index: 0, date: '' };
interface BatchCounts { known: number; active: number; idle: number; unknown: number; players: number }
interface BatchCounts {
known: number;
active: number;
idle: number;
unknown: number;
players: number;
}
function addHeatmap(counts: BatchCounts, start: number, end: number): void {
// Samples are bounded by the shared freshness window. Splitting at UTC minute boundaries
// handles local hour/day boundaries, DST, and fractional-hour zones.
@@ -95,8 +176,13 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
for (let cursor = start; cursor < end;) {
const minute = Math.floor(cursor / 60_000);
if (minute !== cachedMinute) {
const parts = Object.fromEntries(calendar.formatToParts(cursor).map(part => [part.type, part.value]));
localCell = { index: weekdays[parts.weekday] * 24 + Number(parts.hour), date: `${parts.year}-${parts.month}-${parts.day}` };
const parts = Object.fromEntries(
calendar.formatToParts(cursor).map((part) => [part.type, part.value])
);
localCell = {
index: weekdays[parts.weekday] * 24 + Number(parts.hour),
date: `${parts.year}-${parts.month}-${parts.day}`
};
cachedMinute = minute;
}
const stop = Math.min(end, (minute + 1) * 60_000);
@@ -120,11 +206,13 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
const samples = scope.game === null ? batch : batch.filter(matchesScope);
if (!samples.length) return;
const end = Math.min(nextAt, at + ONLINE_MAX_MS, now);
const priorHours = Math.max(0, Math.min(end, previous.until) - Math.max(at, previous.since)) / 3600_000;
const priorHours =
Math.max(0, Math.min(end, previous.until) - Math.max(at, previous.since)) / 3600_000;
if (priorHours > 0) {
previous.firstSampleAt ??= at;
previous.lastSampleAt = at;
for (const sample of samples) addObservation(previous.summary, sample.players, priorHours);
for (const sample of samples)
addObservation(previous.summary, sample.players, priorHours);
}
// Preserve current-window zero-duration rows without bringing rows from
// the separate comparison window into the visible server/map tables.
@@ -146,14 +234,27 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
for (const sample of samples) {
let server = serverTotals.get(sample.server);
if (!server) {
server = { server: sample.server, currentRegistered: registered.has(sample.server), ...emptyUtilization() };
server = {
server: sample.server,
currentRegistered: registered.has(sample.server),
...emptyUtilization()
};
serverTotals.set(sample.server, server);
}
let game = mapTotals.get(sample.game);
if (!game) { game = new Map(); mapTotals.set(sample.game, game); }
if (!game) {
game = new Map();
mapTotals.set(sample.game, game);
}
let map = game.get(sample.map);
if (!map) {
map = { game: sample.game, map: sample.map, full: 0, capacityObserved: 0, ...emptyUtilization() };
map = {
game: sample.game,
map: sample.map,
full: 0,
capacityObserved: 0,
...emptyUtilization()
};
game.set(sample.map, map);
}
addObservation(summary, sample.players, hours);
@@ -167,10 +268,14 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
if (sample.players > 0) counts.active++;
else if (sample.players === 0) counts.idle++;
}
const known = sample.players !== null && sample.capacity !== null && sample.capacity > 0;
const known =
sample.players !== null && sample.capacity !== null && sample.capacity > 0;
const full = known && sample.players! >= sample.capacity!;
const previous = pressure.get(map);
pressure.set(map, { known: known && (previous?.known ?? true), full: full && (previous?.full ?? true) });
pressure.set(map, {
known: known && (previous?.known ?? true),
full: full && (previous?.full ?? true)
});
}
for (const [map, state] of pressure) {
if (state.known) {
@@ -181,9 +286,21 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
addHeatmap(counts, start, end);
}
type SampleRow = [at: number, server: string, game: string, map: string, players: number | null, capacity: number | null];
const samples = handle.prepare(`SELECT at, server, game, map, players, capacity FROM metric_samples
WHERE at >= ? AND at <= ? ORDER BY at`).raw().iterate(previous.since - ONLINE_MAX_MS, now) as IterableIterator<SampleRow>;
type SampleRow = [
at: number,
server: string,
game: string,
map: string,
players: number | null,
capacity: number | null
];
const samples = handle
.prepare(
`SELECT at, server, game, map, players, capacity FROM metric_samples
WHERE at >= ? AND at <= ? ORDER BY at`
)
.raw()
.iterate(previous.since - ONLINE_MAX_MS, now) as IterableIterator<SampleRow>;
for (const [at, server, game, map, players, capacity] of samples) {
if (batch.length && at !== batch[0].at) {
aggregateBatch(at);
@@ -193,16 +310,46 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
}
aggregateBatch(now);
const compareText = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0;
const servers = [...serverTotals.values()].sort((a, b) => b.active - a.active || compareText(a.server, b.server));
const maps = [...mapTotals.values()].flatMap(game => [...game.values()])
.sort((a, b) => b.playerHours - a.playerHours || compareText(a.map, b.map) || compareText(a.game, b.game));
previous.partialHistory = scopedBounds.first === null || scopedBounds.first > previous.since || previous.firstSampleAt === null;
for (let index = 0; index < cells.length; index++) cells[index].distinctDates = cellDates[index].size;
const compareText = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);
const servers = [...serverTotals.values()].sort(
(a, b) => b.active - a.active || compareText(a.server, b.server)
);
const maps = [...mapTotals.values()]
.flatMap((game) => [...game.values()])
.sort(
(a, b) =>
b.playerHours - a.playerHours ||
compareText(a.map, b.map) ||
compareText(a.game, b.game)
);
previous.partialHistory =
scopedBounds.first === null ||
scopedBounds.first > previous.since ||
previous.firstSampleAt === null;
for (let index = 0; index < cells.length; index++)
cells[index].distinctDates = cellDates[index].size;
const fullMapHours = maps.reduce((total, map) => total + map.full, 0);
const knownMapHours = maps.reduce((total, map) => total + map.capacityObserved, 0);
const pooledPressure = { value: knownMapHours > 0 ? 100 * fullMapHours / knownMapHours : null,
fullMapHours, knownMapHours, coveredMaps: maps.filter(map => map.capacityObserved > 0).length, totalMaps: maps.length };
return { days, since, now, ...metadata, scope, scopeOptions, summary, servers, maps, points, previous,
heatmap: { timeZone: scope.timeZone, cells }, pooledPressure };
const pooledPressure = {
value: knownMapHours > 0 ? (100 * fullMapHours) / knownMapHours : null,
fullMapHours,
knownMapHours,
coveredMaps: maps.filter((map) => map.capacityObserved > 0).length,
totalMaps: maps.length
};
return {
days,
since,
now,
...metadata,
scope,
scopeOptions,
summary,
servers,
maps,
points,
previous,
heatmap: { timeZone: scope.timeZone, cells },
pooledPressure
};
}
+133 -38
View File
@@ -10,9 +10,22 @@ import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { serverObservation } from './status.js';
import { recoverySettling } from '../provisioning/restarts.js';
const nativeFields = ['id','server','host_id','generation','library_id','image','bind_address','port','public_address','public_port','created_at','retired_at'];
const nativeFields = [
'id',
'server',
'host_id',
'generation',
'library_id',
'image',
'bind_address',
'port',
'public_address',
'public_port',
'created_at',
'retired_at'
];
const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS instance_generation,n.library_id AS game_library_id,
CASE WHEN n.id IS NOT NULL THEN json_object(${nativeFields.map(key => `'${key}',n.${key}`).join(',')}) END AS instance_json,
CASE WHEN n.id IS NOT NULL THEN json_object(${nativeFields.map((key) => `'${key}',n.${key}`).join(',')}) END AS instance_json,
h.inventory AS host_inventory,h.last_heartbeat AS host_heartbeat,
EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id=h.id AND c.revoked_at IS NULL) AS credential_active,
(SELECT json_object('id',d.id,'action',d.action,'state',d.state,'submitted_at',d.submitted_at)
@@ -27,65 +40,147 @@ const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS in
LEFT JOIN ec_hosts h ON h.id=n.host_id AND h.deleted_at IS NULL
LEFT JOIN restart_rollout r ON r.server=s.name LEFT JOIN server_telemetry t ON t.server=s.name`;
interface Row extends ServerRow {
instance_json: string | null; host_inventory: string | null; host_heartbeat: string | null; credential_active: number;
lifecycle_json: string | null; lifecycle_at: string | null; submission: string | null;
restart_at: string | null; restart_healthy: string | null; restart_operation_id: string | null; telemetry_at: string | null; telemetry_json: string | null;
instance_json: string | null;
host_inventory: string | null;
host_heartbeat: string | null;
credential_active: number;
lifecycle_json: string | null;
lifecycle_at: string | null;
submission: string | null;
restart_at: string | null;
restart_healthy: string | null;
restart_operation_id: string | null;
telemetry_at: string | null;
telemetry_json: string | null;
}
interface Record {
server: ServerRow; native: NativeInstance | null;
server: ServerRow;
native: NativeInstance | null;
host: Pick<Host, 'connected' | 'connectionState' | 'inventory'> | null;
lifecycle: { id: string; action: string; state: string; submitted_at: string | null } | null;
lifecycleAt: string | null; submission: string | null; telemetry: GameTelemetry | null;
restartAt: string | null; restartHealthy: string | null; restartOperationId: string | null;
lifecycleAt: string | null;
submission: string | null;
telemetry: GameTelemetry | null;
restartAt: string | null;
restartHealthy: string | null;
restartOperationId: string | null;
}
/** A pass shares its reads; each asynchronous evidence boundary refreshes in one joined query. */
export class MonitoringSnapshot {
private records = new Map<string, Record>();
constructor(serverName?: string) { ensureHostSchema(db()); this.refresh(serverName); }
constructor(serverName?: string) {
ensureHostSchema(db());
this.refresh(serverName);
}
refresh(server?: string): void {
const rows = server === undefined ? db().prepare(sql + ' WHERE s.destroyed_at IS NULL ORDER BY s.name').all() as Row[]
: db().prepare(sql + ' WHERE s.name=? AND s.destroyed_at IS NULL').all(server) as Row[];
if (server === undefined) this.records.clear(); else this.records.delete(server);
const rows =
server === undefined
? (db()
.prepare(sql + ' WHERE s.destroyed_at IS NULL ORDER BY s.name')
.all() as Row[])
: (db()
.prepare(sql + ' WHERE s.name=? AND s.destroyed_at IS NULL')
.all(server) as Row[]);
if (server === undefined) this.records.clear();
else this.records.delete(server);
for (const row of rows) {
const { instance_json,host_inventory,host_heartbeat,credential_active,lifecycle_json,lifecycle_at,submission,
restart_at,restart_healthy,restart_operation_id,telemetry_at,telemetry_json,...fields } = row;
const connected = !!credential_active && !!host_heartbeat && Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
const {
instance_json,
host_inventory,
host_heartbeat,
credential_active,
lifecycle_json,
lifecycle_at,
submission,
restart_at,
restart_healthy,
restart_operation_id,
telemetry_at,
telemetry_json,
...fields
} = row;
const connected =
!!credential_active &&
!!host_heartbeat &&
Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
let telemetry: GameTelemetry | null = null;
if (telemetry_json) try {
const saved = JSON.parse(telemetry_json) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.name) telemetry = { ...event, receivedAt: telemetry_at!, timing: saved.timing };
} catch { /* Invalid telemetry cannot establish health. */ }
this.records.set(row.name, { server: fields, native: instance_json ? JSON.parse(instance_json) : null,
host: host_inventory ? { connected, connectionState: connected ? 'online' : 'offline',
inventory: parseHostInventory(row.host_id!, host_inventory) } : null,
lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null, lifecycleAt: lifecycle_at, submission, telemetry,
restartAt: restart_at, restartHealthy: restart_healthy, restartOperationId: restart_operation_id });
if (telemetry_json)
try {
const saved = JSON.parse(telemetry_json) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.name)
telemetry = { ...event, receivedAt: telemetry_at!, timing: saved.timing };
} catch {
/* Invalid telemetry cannot establish health. */
}
this.records.set(row.name, {
server: fields,
native: instance_json ? JSON.parse(instance_json) : null,
host: host_inventory
? {
connected,
connectionState: connected ? 'online' : 'offline',
inventory: parseHostInventory(row.host_id!, host_inventory)
}
: null,
lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null,
lifecycleAt: lifecycle_at,
submission,
telemetry,
restartAt: restart_at,
restartHealthy: restart_healthy,
restartOperationId: restart_operation_id
});
}
}
get(server: string) { return this.records.get(server); }
servers() { return [...this.records.values()].map(record => record.server); }
get(server: string) {
return this.records.get(server);
}
servers() {
return [...this.records.values()].map((record) => record.server);
}
provisioning(server: string) {
const operation = this.get(server)?.lifecycle;
return !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state);
return (
!!operation &&
['create', 'rebuild', 'start'].includes(operation.action) &&
['queued', 'running', 'awaiting-health'].includes(operation.state)
);
}
settling(server: string) {
const record = this.get(server);
return recoverySettling(record?.restartAt ? { requested_at: record.restartAt,
healthy_at: record.restartHealthy, operation_id: record.restartOperationId } : undefined, record?.lifecycle);
return recoverySettling(
record?.restartAt
? {
requested_at: record.restartAt,
healthy_at: record.restartHealthy,
operation_id: record.restartOperationId
}
: undefined,
record?.lifecycle
);
}
telemetryEvidence(server: string) {
const record = this.get(server), report = record?.telemetry;
const record = this.get(server),
report = record?.telemetry;
if (!report) return null;
const received = Date.parse(report.receivedAt), age = Date.now() - received;
return Number.isFinite(age) && age >= 0 && age <= ONLINE_MAX_MS
&& (!record?.submission || received > Date.parse(record.submission)) ? report : null;
const received = Date.parse(report.receivedAt),
age = Date.now() - received;
return Number.isFinite(age) &&
age >= 0 &&
age <= ONLINE_MAX_MS &&
(!record?.submission || received > Date.parse(record.submission))
? report
: null;
}
observe = (server: ServerRow) => serverObservation(server, {
provisioning: this.provisioning(server.name), settling: this.settling(server.name),
submission: this.get(server.name)?.submission ?? null, telemetry: this.get(server.name)?.telemetry ?? null
});
observe = (server: ServerRow) =>
serverObservation(server, {
provisioning: this.provisioning(server.name),
settling: this.settling(server.name),
submission: this.get(server.name)?.submission ?? null,
telemetry: this.get(server.name)?.telemetry ?? null
});
patch(server: string, fields: Partial<ServerRow>) {
const record = this.get(server);
if (record) record.server = { ...record.server, ...fields };
+176 -54
View File
@@ -15,90 +15,212 @@ export function serverOperations(server: string, now = Date.now()): FleetOperati
function readOperations(now: number, server?: string): FleetOperation[] {
const since = new Date(now - 86_400_000).toISOString();
const bindings = { server: server ?? null, since };
const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`;
const scope = (column: string) => (server === undefined ? '' : `AND ${column} = @server`);
const handle = db();
const operations: FleetOperation[] = [];
const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`)
const slots = handle
.prepare(
`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`
)
.all(bindings) as { server: string; requested_at: string; error: string | null }[];
const recovering = new Map(slots.map(slot => [slot.server, slot]));
const recovering = new Map(slots.map((slot) => [slot.server, slot]));
const representedRecovery = new Set<string>();
const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
const campaigns = handle
.prepare(
`SELECT d.id, d.action, d.started_at, ds.server,
ds.restart_state, ds.error, ds.restarted_at, ds.players
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as {
id: string; action: string; started_at: string; server: string; restart_state: string;
error: string | null; restarted_at: string | null; players: number | null;
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`
)
.all(bindings) as {
id: string;
action: string;
started_at: string;
server: string;
restart_state: string;
error: string | null;
restarted_at: string | null;
players: number | null;
}[];
for (const row of campaigns) {
const state = row.restart_state === 'done' ? 'done'
: ['failed', 'manual'].includes(row.restart_state) ? 'attention' : 'active';
const state =
row.restart_state === 'done'
? 'done'
: ['failed', 'manual'].includes(row.restart_state)
? 'attention'
: 'active';
const slot = recovering.get(row.server);
if (row.restarted_at && state !== 'done') representedRecovery.add(row.server);
const label = state === 'done' ? 'Update complete' : state === 'attention' ? 'Update needs attention'
: row.restart_state === 'held' ? 'Held by a beta'
: row.restarted_at ? 'Recovering' : (row.players ?? 0) > 0 ? 'Waiting for an empty lobby' : 'Queued for restart';
operations.push({ key: `campaign:${row.id}:${row.server}`, group: `campaign:${row.id}`,
kind: 'campaign', server: row.server, label, state, startedAt: row.started_at,
detail: state === 'done' ? 'Update applied and restart checks completed' : row.error ?? (row.restarted_at ? slot?.error ?? 'Verifying server health'
: (row.players ?? 0) > 0 ? `${row.players} players at the last check`
: 'Waiting for restart eligibility and available capacity'),
href: `/campaigns/${encodeURIComponent(row.id)}` });
const label =
state === 'done'
? 'Update complete'
: state === 'attention'
? 'Update needs attention'
: row.restart_state === 'held'
? 'Held by a beta'
: row.restarted_at
? 'Recovering'
: (row.players ?? 0) > 0
? 'Waiting for an empty lobby'
: 'Queued for restart';
operations.push({
key: `campaign:${row.id}:${row.server}`,
group: `campaign:${row.id}`,
kind: 'campaign',
server: row.server,
label,
state,
startedAt: row.started_at,
detail:
state === 'done'
? 'Update applied and restart checks completed'
: (row.error ??
(row.restarted_at
? (slot?.error ?? 'Verifying server health')
: (row.players ?? 0) > 0
? `${row.players} players at the last check`
: 'Waiting for restart eligibility and available capacity')),
href: `/campaigns/${encodeURIComponent(row.id)}`
});
}
// Only the newest lifecycle operation can still require action on a server.
// Separate branches use state/rowid indexes for fleet reads and the server
// index for detail history. Completed work is never loaded for the full fleet.
const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
const lifecycle = handle
.prepare(
`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
${server === undefined ? '' : `UNION
${
server === undefined
? ''
: `UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND server = @server
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`}
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`
}
UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`)
.all(bindings) as { id: string; server: string; action: string; state: string; stage: string;
error: string | null; started_at: string }[];
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`
)
.all(bindings) as {
id: string;
server: string;
action: string;
state: string;
stage: string;
error: string | null;
started_at: string;
}[];
for (const row of lifecycle) {
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
const state = ['done', 'warning', 'cancelled'].includes(row.state) ? 'done'
: ['failed', 'interrupted'].includes(row.state) ? 'attention' : 'active';
operations.push({ key: `deployment:${row.id}`, group: `deployment:${row.id}`, kind: 'deployment',
server: row.server, state, startedAt: row.started_at,
activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`, resultState: row.state,
label: state === 'done' ? row.state === 'cancelled' ? 'Removal cancelled' : row.state === 'warning' ? 'Completed with warnings' : 'Complete'
: state === 'attention' ? `${actionLabels[row.action as ProvisioningAction]} needs attention` : row.stage,
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
const state = ['done', 'warning', 'cancelled'].includes(row.state)
? 'done'
: ['failed', 'interrupted'].includes(row.state)
? 'attention'
: 'active';
operations.push({
key: `deployment:${row.id}`,
group: `deployment:${row.id}`,
kind: 'deployment',
server: row.server,
state,
startedAt: row.started_at,
activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`,
resultState: row.state,
label:
state === 'done'
? row.state === 'cancelled'
? 'Removal cancelled'
: row.state === 'warning'
? 'Completed with warnings'
: 'Complete'
: state === 'attention'
? `${actionLabels[row.action as ProvisioningAction]} needs attention`
: row.stage,
detail: row.error ?? row.stage,
href: `/provisioning/${encodeURIComponent(row.id)}`
});
}
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`)
.all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' };
const betas = handle
.prepare(
`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`
)
.all(bindings) as {
id: number;
opened_at: string;
server: string;
stage: string;
error: string | null;
}[];
const stages: Record<string, string> = {
queued: 'Waiting for an earlier test wave',
deploying: 'Deploying beta',
'awaiting-empty': 'Waiting for an empty lobby',
restarting: 'Restarting',
'awaiting-health': 'Recovering',
'awaiting-match': 'Observing a match',
ready: 'Ready for review',
failed: 'Beta needs attention'
};
for (const row of betas) {
if (['restarting', 'awaiting-health'].includes(row.stage)) representedRecovery.add(row.server);
operations.push({ key: `beta:${row.id}:${row.server}`, group: `beta:${row.id}`,
kind: 'beta', server: row.server, startedAt: row.opened_at,
state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
if (['restarting', 'awaiting-health'].includes(row.stage))
representedRecovery.add(row.server);
operations.push({
key: `beta:${row.id}:${row.server}`,
group: `beta:${row.id}`,
kind: 'beta',
server: row.server,
startedAt: row.opened_at,
state: row.stage === 'failed' ? 'attention' : 'active',
label: stages[row.stage] ?? row.stage,
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`,
href: `/betas/${row.id}`
});
}
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
.all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
const maintenance = handle
.prepare(
`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`
)
.all(bindings) as {
id: number;
server: string;
kind: string;
detail: string | null;
requested_at: string;
}[];
for (const job of maintenance) {
operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance',
server: job.server, state: 'attention', startedAt: job.requested_at,
label: job.kind === 'update' ? 'Update verification needs attention' : 'Maintenance needs attention',
detail: job.detail ?? 'Inspect the maintenance result', href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}` });
operations.push({
key: `maintenance:${job.id}`,
group: `maintenance:${job.id}`,
kind: 'maintenance',
server: job.server,
state: 'attention',
startedAt: job.requested_at,
label:
job.kind === 'update'
? 'Update verification needs attention'
: 'Maintenance needs attention',
detail: job.detail ?? 'Inspect the maintenance result',
href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}`
});
}
for (const slot of slots) {
if (representedRecovery.has(slot.server)) continue;
operations.push({ key: `recovery:${slot.server}:${slot.requested_at}`, group: `recovery:${slot.server}`,
kind: 'recovery', server: slot.server, startedAt: slot.requested_at, state: slot.error ? 'attention' : 'active',
label: 'Recovering from restart', detail: slot.error ?? 'Verifying server health',
href: `/servers/${encodeURIComponent(slot.server)}` });
operations.push({
key: `recovery:${slot.server}:${slot.requested_at}`,
group: `recovery:${slot.server}`,
kind: 'recovery',
server: slot.server,
startedAt: slot.requested_at,
state: slot.error ? 'attention' : 'active',
label: 'Recovering from restart',
detail: slot.error ?? 'Verifying server health',
href: `/servers/${encodeURIComponent(slot.server)}`
});
}
return operations.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
}
+4 -1
View File
@@ -3,7 +3,10 @@ import { monitoringStatus } from './health-observations.js';
import { MonitoringSnapshot } from './monitoring-snapshot.js';
/** Read only the accepted, current-instance sample; never dispatch a probe. */
export function observedPlayerCapacity(server: string, snapshot: MonitoringSnapshot): PlayerCapacityObservation {
export function observedPlayerCapacity(
server: string,
snapshot: MonitoringSnapshot
): PlayerCapacityObservation {
const game = monitoringStatus(server, snapshot)?.game;
const capacity = game?.result.ok ? game.result.status.capacity : null;
return typeof capacity === 'number' && Number.isSafeInteger(capacity) && capacity > 0
+70 -23
View File
@@ -2,7 +2,13 @@ import { db } from '../db/index.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
import { getPersonalBestRecords } from './leaderboard-records.js';
import { countsAsGamePlayed } from './leaderboard-participation.js';
import type { LeaderboardOptions, PlayerProfile, PlayerStats, PlayerMapRecord, PlayerMatch } from '../../leaderboard.js';
import type {
LeaderboardOptions,
PlayerProfile,
PlayerStats,
PlayerMapRecord,
PlayerMatch
} from '../../leaderboard.js';
const totals = `MAX(mp.highest_round) AS highestRound, SUM(mp.kills) AS kills,
SUM(mp.rounds_survived) AS roundsSurvived, SUM(mp.headshots) AS headshots,
@@ -12,38 +18,68 @@ const scope = `FROM leaderboard_match_players mp JOIN leaderboard_matches m ON m
WHERE m.game IN ('t5', 't6') AND m.status = 'completed' AND mp.guid = ? AND mp.has_stats = 1
AND m.highest_round >= ${MIN_RECORDED_ROUND} AND mp.highest_round >= ${MIN_RECORDED_ROUND}`;
export function getPlayerProfile(guid: string, requestedOffset = 0,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players'> = {}): PlayerProfile | null {
export function getPlayerProfile(
guid: string,
requestedOffset = 0,
options: Pick<LeaderboardOptions, 'game' | 'map' | 'players'> = {}
): PlayerProfile | null {
const handle = db();
// Keep game-specific ingestion identities, but show one identity with the latest reported name.
const player = handle.prepare(`SELECT guid, name, MIN(first_seen_at) OVER () AS firstSeen,
const player = handle
.prepare(
`SELECT guid, name, MIN(first_seen_at) OVER () AS firstSeen,
MAX(last_seen_at) OVER () AS lastSeen FROM leaderboard_players
WHERE game IN ('t5', 't6') AND guid = ? ORDER BY last_seen_at DESC, game LIMIT 1`)
WHERE game IN ('t5', 't6') AND guid = ? ORDER BY last_seen_at DESC, game LIMIT 1`
)
.get(guid) as Pick<PlayerProfile, 'guid' | 'name' | 'firstSeen' | 'lastSeen'> | undefined;
if (!player) return null;
const { matchCount, ...stats } = handle.prepare(`SELECT COUNT(*) AS matchCount, ${totals} ${scope}`)
const { matchCount, ...stats } = handle
.prepare(`SELECT COUNT(*) AS matchCount, ${totals} ${scope}`)
.get(guid) as PlayerStats & { matchCount: number };
if (!matchCount) return null;
const historyParameters: Array<string | number> = [guid];
let historyScope = scope;
if (options.game) { historyScope += ' AND m.game = ?'; historyParameters.push(options.game); }
if (options.map) { historyScope += ' AND m.map_key = ?'; historyParameters.push(options.map); }
if (options.players) { historyScope += ' AND m.start_players = ?'; historyParameters.push(options.players); }
const matchesTotal = options.game || options.map || options.players
? (handle.prepare(`SELECT COUNT(*) AS total ${historyScope}`).get(...historyParameters) as { total: number }).total
: matchCount;
if (options.game) {
historyScope += ' AND m.game = ?';
historyParameters.push(options.game);
}
if (options.map) {
historyScope += ' AND m.map_key = ?';
historyParameters.push(options.map);
}
if (options.players) {
historyScope += ' AND m.start_players = ?';
historyParameters.push(options.players);
}
const matchesTotal =
options.game || options.map || options.players
? (
handle
.prepare(`SELECT COUNT(*) AS total ${historyScope}`)
.get(...historyParameters) as { total: number }
).total
: matchCount;
const limit = 25;
const lastOffset = Math.max(0, Math.floor((matchesTotal - 1) / limit) * limit);
const offset = Number.isSafeInteger(requestedOffset) ? Math.min(lastOffset, Math.max(0, requestedOffset)) : 0;
const mapStats = handle.prepare(`SELECT m.game, m.map_key AS key, m.map_name AS name,
const offset = Number.isSafeInteger(requestedOffset)
? Math.min(lastOffset, Math.max(0, requestedOffset))
: 0;
const mapStats = handle
.prepare(
`SELECT m.game, m.map_key AS key, m.map_name AS name,
MAX(mp.last_seen_at) AS lastSeen, ${totals} ${scope}
GROUP BY m.game, m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, m.game, key`)
GROUP BY m.game, m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, m.game, key`
)
.all(guid) as Array<Omit<PlayerMapRecord, 'records' | 'ranks'>>;
const [records, ...mapRecords] = getPersonalBestRecords(handle, [{ guid, ...stats },
...mapStats.map(map => ({ guid, ...map, map: map.key }))]);
const [records, ...mapRecords] = getPersonalBestRecords(handle, [
{ guid, ...stats },
...mapStats.map((map) => ({ guid, ...map, map: map.key }))
]);
// Rank all three bests in one pass over this player's maps. Filter the GUID
// only after ranking competitors so ties and skipped ranks match the leaderboard.
const mapRanks = handle.prepare(`WITH totals AS (
const mapRanks = handle
.prepare(
`WITH totals AS (
SELECT requested.key AS requestId, mp.guid,
MAX(mp.highest_round) AS highestRound, MAX(mp.kills) AS bestKills, MAX(mp.best_score) AS bestScore
FROM json_each(?) requested
@@ -60,14 +96,25 @@ export function getPlayerProfile(guid: string, requestedOffset = 0,
RANK() OVER (PARTITION BY requestId ORDER BY bestKills DESC) AS bestKills,
RANK() OVER (PARTITION BY requestId ORDER BY bestScore DESC) AS bestScore
FROM totals
) SELECT highestRound, bestKills, bestScore FROM ranked WHERE guid = ? ORDER BY requestId`)
.all(JSON.stringify(mapStats.map(({ game, key }) => ({ game, key }))), guid) as PlayerMapRecord['ranks'][];
const maps = mapStats.map((map, index) => ({ ...map, records: mapRecords[index], ranks: mapRanks[index] }));
const matches = handle.prepare(`SELECT m.id, m.game, m.server, m.server_deleted_at, m.map_name AS mapName,
) SELECT highestRound, bestKills, bestScore FROM ranked WHERE guid = ? ORDER BY requestId`
)
.all(
JSON.stringify(mapStats.map(({ game, key }) => ({ game, key }))),
guid
) as PlayerMapRecord['ranks'][];
const maps = mapStats.map((map, index) => ({
...map,
records: mapRecords[index],
ranks: mapRanks[index]
}));
const matches = handle
.prepare(
`SELECT m.id, m.game, m.server, m.server_deleted_at, m.map_name AS mapName,
m.started_at AS startedAt, mp.last_seen_at AS lastSeen, m.status, mp.completed,
mp.highest_round AS highestRound, mp.rounds_survived AS roundsSurvived,
mp.kills, mp.headshots, mp.revives, mp.downs, mp.best_score AS bestScore
${historyScope} ORDER BY mp.last_seen_at DESC, m.id DESC LIMIT ? OFFSET ?`)
${historyScope} ORDER BY mp.last_seen_at DESC, m.id DESC LIMIT ? OFFSET ?`
)
.all(...historyParameters, limit, offset) as PlayerMatch[];
return { ...player, stats, records, maps, matches, matchesTotal, offset, limit };
}
+110 -25
View File
@@ -1,13 +1,29 @@
import { createHash } from 'node:crypto';
import { db } from '../db/index.js';
import { createScript, getScript, getVersion, uploadNewVersion, type ScopeType } from './scripts.js';
import {
createScript,
getScript,
getVersion,
uploadNewVersion,
type ScopeType
} from './scripts.js';
import { findRunningBetaForScript } from './beta.js';
import { downloadGithubAsset, getGithubRelease, GithubReleaseError } from '../integrations/github-releases.js';
import {
downloadGithubAsset,
getGithubRelease,
GithubReleaseError
} from '../integrations/github-releases.js';
import type { PluginSource, PluginUpdate } from '$lib/plugin-github.js';
export function pluginSource(pluginId: number): PluginSource | null {
return db().prepare(`SELECT p.* FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
WHERE v.script_id = ? ORDER BY v.version DESC LIMIT 1`).get(pluginId) as PluginSource | undefined ?? null;
return (
(db()
.prepare(
`SELECT p.* FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
WHERE v.script_id = ? ORDER BY v.version DESC LIMIT 1`
)
.get(pluginId) as PluginSource | undefined) ?? null
);
}
export async function checkPluginUpdate(pluginId: number): Promise<PluginUpdate> {
@@ -19,13 +35,27 @@ export async function checkPluginUpdate(pluginId: number): Promise<PluginUpdate>
try {
const release = await getGithubRelease(`https://github.com/${source.repository}`);
// A latest-stable release may already be in history when a newer prerelease is imported.
if (release.id !== source.release_id && Date.parse(release.publishedAt) <= Date.parse(source.published_at)) {
if (
release.id !== source.release_id &&
Date.parse(release.publishedAt) <= Date.parse(source.published_at)
) {
return { ...base, release, status: 'ahead' };
}
const imported = db().prepare(`SELECT p.version_id FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
const imported = db()
.prepare(
`SELECT p.version_id FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id
WHERE v.script_id = ? AND lower(p.repository) = lower(?) AND p.release_id = ? AND p.asset_name = ?
ORDER BY v.version DESC LIMIT 1`).get(pluginId, source.repository, release.id, source.asset_name) as { version_id: number } | undefined;
if (imported) return { ...base, release, status: imported.version_id === plugin.current_version_id ? 'current' : 'imported', versionId: imported.version_id };
ORDER BY v.version DESC LIMIT 1`
)
.get(pluginId, source.repository, release.id, source.asset_name) as
{ version_id: number } | undefined;
if (imported)
return {
...base,
release,
status: imported.version_id === plugin.current_version_id ? 'current' : 'imported',
versionId: imported.version_id
};
return { ...base, release, status: 'available' };
} catch (error) {
return { ...base, release: null, status: 'error', error: (error as Error).message };
@@ -42,44 +72,99 @@ export interface ImportPluginReleaseInput {
}
export async function importPluginRelease(input: ImportPluginReleaseInput) {
if (typeof input.url !== 'string' || !Number.isSafeInteger(input.assetId) || input.assetId < 1) {
if (
typeof input.url !== 'string' ||
!Number.isSafeInteger(input.assetId) ||
input.assetId < 1
) {
throw new GithubReleaseError('A release URL and DLL asset selection are required.');
}
if (input.pluginId !== undefined && (!Number.isSafeInteger(input.pluginId) || input.pluginId < 1)) {
if (
input.pluginId !== undefined &&
(!Number.isSafeInteger(input.pluginId) || input.pluginId < 1)
) {
throw new GithubReleaseError('Invalid plugin id.');
}
const requirePlugin = () => {
const plugin = input.pluginId === undefined ? undefined : getScript(input.pluginId);
if (input.pluginId !== undefined && (!plugin || plugin.kind !== 'plugin')) throw new GithubReleaseError('Plugin not found.', 404);
if (plugin && findRunningBetaForScript(plugin.id)) throw new GithubReleaseError('Finish the running beta before importing another release.', 409);
if (input.pluginId !== undefined && (!plugin || plugin.kind !== 'plugin'))
throw new GithubReleaseError('Plugin not found.', 404);
if (plugin && findRunningBetaForScript(plugin.id))
throw new GithubReleaseError(
'Finish the running beta before importing another release.',
409
);
return plugin;
};
requirePlugin();
if (input.pluginId === undefined && (typeof input.name !== 'string' || !input.name.trim())) throw new GithubReleaseError('A plugin name is required.');
if (input.pluginId === undefined && (typeof input.name !== 'string' || !input.name.trim()))
throw new GithubReleaseError('A plugin name is required.');
// Refetch the exact reviewed release: asset names, notes, tags and digests are never trusted from the client.
const release = await getGithubRelease(input.url, true);
const asset = release.assets.find(a => a.id === input.assetId);
if (!asset) throw new GithubReleaseError('That DLL is no longer available in this release. Preview it again.');
const asset = release.assets.find((a) => a.id === input.assetId);
if (!asset)
throw new GithubReleaseError(
'That DLL is no longer available in this release. Preview it again.'
);
const content = await downloadGithubAsset(release, asset);
const sha = createHash('sha256').update(content).digest('hex');
return db().transaction(() => {
const plugin = requirePlugin();
if (plugin) {
const prior = db().prepare(`SELECT v.id FROM script_versions v JOIN plugin_imports p ON p.version_id = v.id
const prior = db()
.prepare(
`SELECT v.id FROM script_versions v JOIN plugin_imports p ON p.version_id = v.id
WHERE v.script_id = ? AND lower(p.repository) = lower(?) AND p.release_id = ? AND p.asset_id = ? AND v.sha256 = ?
ORDER BY v.version DESC LIMIT 1`).get(plugin.id, release.repository, release.id, asset.id, sha) as { id: number } | undefined;
ORDER BY v.version DESC LIMIT 1`
)
.get(plugin.id, release.repository, release.id, asset.id, sha) as
{ id: number } | undefined;
if (prior) return result(plugin.id, getVersion(prior.id)!, true);
}
const created = plugin ? { script: plugin, version: uploadNewVersion(plugin.id, content, release.notes, release.tag) }
: createScript({ name: input.name!.trim(), kind: 'plugin', targetRelPath: asset.name, content,
releaseVersion: release.tag, notes: release.notes, scopeType: input.scopeType, scopeValue: input.scopeValue });
db().prepare(`INSERT INTO plugin_imports (version_id, repository, release_id, release_url, asset_id, asset_name, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(created.version.id, release.repository, release.id, release.url, asset.id, asset.name, release.publishedAt);
const created = plugin
? {
script: plugin,
version: uploadNewVersion(plugin.id, content, release.notes, release.tag)
}
: createScript({
name: input.name!.trim(),
kind: 'plugin',
targetRelPath: asset.name,
content,
releaseVersion: release.tag,
notes: release.notes,
scopeType: input.scopeType,
scopeValue: input.scopeValue
});
db()
.prepare(
`INSERT INTO plugin_imports (version_id, repository, release_id, release_url, asset_id, asset_name, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`
)
.run(
created.version.id,
release.repository,
release.id,
release.url,
asset.id,
asset.name,
release.publishedAt
);
return result(created.script.id, created.version, false);
})();
}
function result(pluginId: number, version: NonNullable<ReturnType<typeof getVersion>>, alreadyImported: boolean) {
return { pluginId, versionId: version.id, releaseVersion: version.release_version, sha256: version.sha256,
alreadyImported, betaUrl: `/betas?pluginId=${pluginId}&versionId=${version.id}` };
function result(
pluginId: number,
version: NonNullable<ReturnType<typeof getVersion>>,
alreadyImported: boolean
) {
return {
pluginId,
versionId: version.id,
releaseVersion: version.release_version,
sha256: version.sha256,
alreadyImported,
betaUrl: `/betas?pluginId=${pluginId}&versionId=${version.id}`
};
}
+1 -1
View File
@@ -1,2 +1,2 @@
// Compatibility for existing jobs and API clients.
export * from "../maintenance/runtime.js";
export * from '../maintenance/runtime.js';
+12 -4
View File
@@ -1,10 +1,18 @@
import { getVersion, getVersionContent, getVersionMetadata, listScripts } from './scripts.js';
function selectedArtifacts<Version>(game: string | null, map: string | null, readVersion: (id: number) => Version | undefined) {
function selectedArtifacts<Version>(
game: string | null,
map: string | null,
readVersion: (id: number) => Version | undefined
) {
return (['script', 'plugin'] as const).flatMap((kind) =>
listScripts(kind).flatMap((script) => {
const scopeMatches = script.scope_type === 'game' ? script.scope_value === game
: script.scope_type === 'map' ? script.scope_value === map : false;
const scopeMatches =
script.scope_type === 'game'
? script.scope_value === game
: script.scope_type === 'map'
? script.scope_value === map
: false;
if (!scopeMatches || !script.current_version_id) return [];
const version = readVersion(script.current_version_id);
return version ? [{ kind, script, version }] : [];
@@ -14,7 +22,7 @@ function selectedArtifacts<Version>(game: string | null, map: string | null, rea
/** Published file artifacts shared by provisioning and drift reconciliation. */
export function promotedArtifacts(game: string | null, map: string | null) {
return selectedArtifacts(game, map, id => {
return selectedArtifacts(game, map, (id) => {
const version = getVersion(id);
return version ? { ...version, content: getVersionContent(id) } : undefined;
});
+25 -8
View File
@@ -2,17 +2,34 @@ import { db } from '../db/index.js';
/** Maintenance must leave pending lifecycle work and retained recovery to Provisioning. */
export function hasActiveProvisioning(server: string, exceptOperationId = ''): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments
return (
!!db()
.prepare(
`SELECT 1 FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL AND id != ?
AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1`).get(server, exceptOperationId)
|| !!db().prepare(`SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL
AND (operation_id IS NULL OR operation_id != ?)`).get(server, exceptOperationId);
AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1`
)
.get(server, exceptOperationId) ||
!!db()
.prepare(
`SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL
AND (operation_id IS NULL OR operation_id != ?)`
)
.get(server, exceptOperationId)
);
}
/** Only the latest lifecycle operation can describe the current server. */
export function isProvisioning(server: string): boolean {
const operation = db().prepare(`SELECT action, state FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as
{ action: string; state: string } | undefined;
return !!operation && ['create', 'rebuild', 'start'].includes(operation.action) && ['queued', 'running', 'awaiting-health'].includes(operation.state);
const operation = db()
.prepare(
`SELECT action, state FROM server_deployments
WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`
)
.get(server) as { action: string; state: string } | undefined;
return (
!!operation &&
['create', 'rebuild', 'start'].includes(operation.action) &&
['queued', 'running', 'awaiting-health'].includes(operation.state)
);
}
+148 -50
View File
@@ -3,7 +3,12 @@ import type { ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
import { getVersion } from './scripts.js';
import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js';
import {
cancelStagedFiles,
isFileApplied,
queueFileChange,
stagedFileChanges
} from './staged-files.js';
// Roll a promoted deployment back to whatever versions were current before
// the promote landed. Reads deployment_scripts as the source of truth for
@@ -12,7 +17,15 @@ import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges }
// pre-multi-script legacy rows.
export class RollbackError extends Error {
constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back' | 'snapshot-pending') {
constructor(
message: string,
public readonly kind:
| 'history-expired'
| 'no-such-deployment'
| 'not-promote'
| 'already-rolled-back'
| 'snapshot-pending'
) {
super(message);
}
}
@@ -58,31 +71,53 @@ function nowIso(): string {
}
function rollbackDeploymentId(promoteId: string): string {
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
const stamp = new Date()
.toISOString()
.replace(/[-:]/g, '')
.replace(/\.\d+Z$/, 'Z');
return `dep-${stamp}-rollback-${promoteId}`;
}
export interface RollbackDeploymentResult {
deploymentId: string; // the new rollback deployment
reverses: string; // the promote deployment id
scriptIds: number[]; // scripts whose current_version_id got reverted
deploymentId: string; // the new rollback deployment
reverses: string; // the promote deployment id
scriptIds: number[]; // scripts whose current_version_id got reverted
affectedServers: string[];
}
export async function rollbackPromoteDeployment(promoteId: string): Promise<RollbackDeploymentResult> {
export async function rollbackPromoteDeployment(
promoteId: string
): Promise<RollbackDeploymentResult> {
const dep = db()
.prepare(`SELECT id, action, script_id, script_version_id, prior_version_id, rolled_back_by, beta_id
FROM deployments WHERE id = ?`)
.prepare(
`SELECT id, action, script_id, script_version_id, prior_version_id, rolled_back_by, beta_id
FROM deployments WHERE id = ?`
)
.get(promoteId) as DeploymentRow | undefined;
if (!dep) throw new RollbackError(`no such deployment ${promoteId}`, 'no-such-deployment');
if (db().prepare("SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?").get(promoteId)) {
throw new RollbackError('Rollback file contents have expired under the retention policy.', 'history-expired');
if (
db()
.prepare(
"SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?"
)
.get(promoteId)
) {
throw new RollbackError(
'Rollback file contents have expired under the retention policy.',
'history-expired'
);
}
if (dep.action !== 'promote') {
throw new RollbackError(`deployment ${promoteId} is action='${dep.action}', not 'promote'`, 'not-promote');
throw new RollbackError(
`deployment ${promoteId} is action='${dep.action}', not 'promote'`,
'not-promote'
);
}
if (dep.rolled_back_by) {
throw new RollbackError(`deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`, 'already-rolled-back');
throw new RollbackError(
`deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`,
'already-rolled-back'
);
}
// Source of truth for which scripts (and prior versions) this promote
@@ -92,13 +127,15 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
.prepare(`SELECT * FROM deployment_scripts WHERE deployment_id = ? ORDER BY script_id`)
.all(promoteId) as DepScriptRow[];
if (scripts.length === 0 && dep.script_id !== null && dep.script_version_id !== null) {
scripts = [{
deployment_id: promoteId,
script_id: dep.script_id,
script_version_id: dep.script_version_id,
prior_version_id: dep.prior_version_id,
kind: 'script'
}];
scripts = [
{
deployment_id: promoteId,
script_id: dep.script_id,
script_version_id: dep.script_version_id,
prior_version_id: dep.prior_version_id,
kind: 'script'
}
];
}
const fileRows = db()
@@ -132,17 +169,29 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
const staged = stagedFileChanges('campaign', promoteId);
const scriptForFile = (file: FileRow): DepScriptRow | undefined => {
const candidates = scriptsByTarget.get(file.rel_path) ?? [];
const versionId = staged.find(row => row.deployment_file_id === file.id)?.script_version_id;
if (versionId) return candidates.find(script => script.script_version_id === versionId);
const versionId = staged.find(
(row) => row.deployment_file_id === file.id
)?.script_version_id;
if (versionId) return candidates.find((script) => script.script_version_id === versionId);
if (dep.beta_id) {
const participant = candidates.find(script => db().prepare(`SELECT 1 FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`).get(dep.beta_id, file.server, script.script_id));
const participant = candidates.find((script) =>
db()
.prepare(
`SELECT 1 FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`
)
.get(dep.beta_id, file.server, script.script_id)
);
if (participant) return participant;
}
return candidates.find(script => getVersion(script.script_version_id)?.sha256 === file.sha256_after) ?? candidates[0];
return (
candidates.find(
(script) => getVersion(script.script_version_id)?.sha256 === file.sha256_after
) ?? candidates[0]
);
};
const kindForFile = (file: FileRow): ArtifactKind => {
const stagedKind = staged.find(row => row.deployment_file_id === file.id)?.kind;
const stagedKind = staged.find((row) => row.deployment_file_id === file.id)?.kind;
if (stagedKind) return stagedKind;
const owner = scriptForFile(file);
if (owner) return owner.kind ?? kindByScriptId.get(owner.script_id) ?? 'script';
@@ -164,11 +213,23 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
// must not restore or remove files that it never installed. Rows from
// older releases have no staging record and retain their audit meaning.
const kind = kindForFile(f);
const preservesBetaSnapshot = staged.some(row => row.server === f.server && row.kind === kind
&& row.rel_path === f.rel_path && row.preserve_before_image);
if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue;
const preservesBetaSnapshot = staged.some(
(row) =>
row.server === f.server &&
row.kind === kind &&
row.rel_path === f.rel_path &&
row.preserve_before_image
);
if (
!preservesBetaSnapshot &&
isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false
)
continue;
if (f.snapshot_pending) {
throw new RollbackError(`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`, 'snapshot-pending');
throw new RollbackError(
`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`,
'snapshot-pending'
);
}
const arr = perServer.get(f.server);
if (arr) arr.push(f);
@@ -176,8 +237,13 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
}
const affectedServers: string[] = [];
const active = (server: string) => !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server);
const active = (server: string) =>
!!db()
.prepare(
`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`
)
.get(promoteId, server);
// Write the new rollback deployment + mark the original.
const insertDep = db().prepare(
@@ -204,8 +270,12 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
cancelStagedFiles('campaign', promoteId);
// Withdrawing this source leaves a shared restart available to other
// campaigns. Provisioning cancels an operation with no eligible sources.
db().prepare(`UPDATE deployment_servers SET restart_state = 'skipped', error = NULL
WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).run(promoteId);
db()
.prepare(
`UPDATE deployment_servers SET restart_state = 'skipped', error = NULL
WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`
)
.run(promoteId);
insertDep.run(newDepId, startedAt);
// Mirror the promote's script list — swapped: the "current" version
// after rollback is the promote's prior; the "prior" of the rollback
@@ -221,16 +291,20 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
}
for (const [server, files] of perServer) {
if (!active(server)) continue;
if (files.some(file => file.verb !== 'added' && file.content_before === null)) {
insertDepServer.run(newDepId, server, 0, 'failed', 'Original file snapshot is missing; cannot restore safely');
if (files.some((file) => file.verb !== 'added' && file.content_before === null)) {
insertDepServer.run(
newDepId,
server,
0,
'failed',
'Original file snapshot is missing; cannot restore safely'
);
continue;
}
insertDepServer.run(newDepId, server, files.length, 'pending', null);
for (const f of files) {
const invertedVerb =
f.verb === 'added' ? 'removed'
: f.verb === 'removed' ? 'added'
: 'updated';
f.verb === 'added' ? 'removed' : f.verb === 'removed' ? 'added' : 'updated';
const inserted = insertDepFile.run(
newDepId,
server,
@@ -242,27 +316,51 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
putFileBlob(f.content_before)
);
const owningScript = scriptForFile(f);
const betaPrior = dep.beta_id && owningScript ? db().prepare(`SELECT prior_version_id FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`).get(dep.beta_id, server, owningScript.script_id) as
{ prior_version_id: number | null } | undefined : undefined;
const priorVersionId = betaPrior ? betaPrior.prior_version_id : owningScript?.prior_version_id;
const betaPrior =
dep.beta_id && owningScript
? (db()
.prepare(
`SELECT prior_version_id FROM beta_server_scripts
WHERE beta_id = ? AND server = ? AND script_id = ?`
)
.get(dep.beta_id, server, owningScript.script_id) as
{ prior_version_id: number | null } | undefined)
: undefined;
const priorVersionId = betaPrior
? betaPrior.prior_version_id
: owningScript?.prior_version_id;
const priorVersion = priorVersionId ? getVersion(priorVersionId) : null;
queueFileChange({ sourceKind: 'campaign', sourceId: newDepId, server,
kind: kindForFile(f), relPath: f.rel_path,
queueFileChange({
sourceKind: 'campaign',
sourceId: newDepId,
server,
kind: kindForFile(f),
relPath: f.rel_path,
content: f.verb === 'added' ? null : f.content_before,
scriptVersionId: f.verb === 'updated' ? priorVersion?.id ?? null : null,
inventorySource: `rollback:${newDepId}`, deploymentFileId: Number(inserted.lastInsertRowid) });
scriptVersionId: f.verb === 'updated' ? (priorVersion?.id ?? null) : null,
inventorySource: `rollback:${newDepId}`,
deploymentFileId: Number(inserted.lastInsertRowid)
});
}
affectedServers.push(server);
}
db().prepare(`UPDATE deployments SET rolled_back_by = ? WHERE id = ?`).run(newDepId, promoteId);
db()
.prepare(`UPDATE deployments SET rolled_back_by = ? WHERE id = ?`)
.run(newDepId, promoteId);
// Restore each affected script's current_version_id.
for (const s of scripts) {
db().prepare(`UPDATE scripts SET current_version_id = ? WHERE id = ?`).run(s.prior_version_id, s.script_id);
db()
.prepare(`UPDATE scripts SET current_version_id = ? WHERE id = ?`)
.run(s.prior_version_id, s.script_id);
}
})();
emit({ type: 'deployment-opened', deploymentId: newDepId, action: 'rollback', servers: affectedServers });
emit({
type: 'deployment-opened',
deploymentId: newDepId,
action: 'rollback',
servers: affectedServers
});
return {
deploymentId: newDepId,
+6 -1
View File
@@ -19,7 +19,12 @@ export function applyRuntimePassword(text: string, password?: string | null): st
}
if (!matched) {
if (lines.at(-1) === '') lines.pop();
lines.push('', '// Added by Plutainer from PLUTAINER_RCON_PASSWORD.', `set rcon_password ${replacement}`, '');
lines.push(
'',
'// Added by Plutainer from PLUTAINER_RCON_PASSWORD.',
`set rcon_password ${replacement}`,
''
);
changed = true;
}
return changed ? lines.join('\n') : text;
+137 -35
View File
@@ -5,7 +5,12 @@ import { configFileForGame, type ArtifactKind } from '../config.js';
export type { ArtifactKind };
export const ARTIFACT_KINDS: ReadonlyArray<ArtifactKind> = ['script', 'config', 'plugin', 'baseline'];
export const ARTIFACT_KINDS: ReadonlyArray<ArtifactKind> = [
'script',
'config',
'plugin',
'baseline'
];
export function isArtifactKind(v: unknown): v is ArtifactKind {
return typeof v === 'string' && (ARTIFACT_KINDS as readonly string[]).includes(v);
@@ -78,8 +83,12 @@ function sha256(buf: Buffer): string {
function normalizeReleaseVersion(value: string | null | undefined): string | null {
if (value == null) return null;
// eslint-disable-next-line no-control-regex -- Reject control characters in release versions.
if (typeof value !== 'string' || value.length > 80 || /[\u0000-\u001f\u007f]/.test(value)) {
throw new ScriptError('release version must be a single line of at most 80 characters', 'invalid-release-version');
throw new ScriptError(
'release version must be a single line of at most 80 characters',
'invalid-release-version'
);
}
return value.trim() || null;
}
@@ -94,7 +103,9 @@ export function getScriptByName(name: string): ScriptRow | undefined {
export function listScripts(kind?: ArtifactKind): ScriptRow[] {
if (kind) {
return db().prepare('SELECT * FROM scripts WHERE kind = ? ORDER BY name').all(kind) as ScriptRow[];
return db()
.prepare('SELECT * FROM scripts WHERE kind = ? ORDER BY name')
.all(kind) as ScriptRow[];
}
return db().prepare('SELECT * FROM scripts ORDER BY name').all() as ScriptRow[];
}
@@ -104,7 +115,10 @@ export function listScripts(kind?: ArtifactKind): ScriptRow[] {
// gated through the beta pipeline. `includeUntested` returns the baseline row
// even without a promoted version (used at beta-open time when the baseline is
// being introduced in the same beta as the overlays).
export function getBaselineForScope(scopeType: ScopeType, scopeValue: string): ScriptRow | undefined {
export function getBaselineForScope(
scopeType: ScopeType,
scopeValue: string
): ScriptRow | undefined {
return db()
.prepare(
`SELECT * FROM scripts
@@ -137,40 +151,49 @@ export function listPromotedOverlaysForScope(
return rows.filter((r) => !skip.has(r.id));
}
export const VERSION_COLUMNS = 'id, script_id, version, release_version, sha256, size, notes, state, kind, uploaded_at';
export const VERSION_COLUMNS =
'id, script_id, version, release_version, sha256, size, notes, state, kind, uploaded_at';
export function getVersionContent(id: number): Buffer {
const row = db().prepare(`SELECT ${blobContent('content_hash')} AS content FROM script_versions WHERE id = ?`).get(id) as { content: Buffer } | undefined;
if (!row?.content) throw new ScriptError(`version content is unavailable (id=${id})`, 'no-such-version');
const row = db()
.prepare(
`SELECT ${blobContent('content_hash')} AS content FROM script_versions WHERE id = ?`
)
.get(id) as { content: Buffer } | undefined;
if (!row?.content)
throw new ScriptError(`version content is unavailable (id=${id})`, 'no-such-version');
return row.content;
}
export function listVersions(scriptId: number): ScriptVersionRow[] {
return db()
.prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC`)
.prepare(
`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC`
)
.all(scriptId) as ScriptVersionRow[];
}
export function getVersion(id: number): ScriptVersionRow | undefined {
return db().prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE id = ?`).get(id) as
| ScriptVersionRow
| undefined;
ScriptVersionRow | undefined;
}
export type ScriptVersionMetadata = Pick<ScriptVersionRow, 'id' | 'version' | 'sha256' | 'size'>;
/** Planning a launch never needs to load potentially large script or plugin bytes. */
export function getVersionMetadata(id: number): ScriptVersionMetadata | undefined {
return db().prepare('SELECT id, version, sha256, size FROM script_versions WHERE id = ?').get(id) as
| ScriptVersionMetadata
| undefined;
return db()
.prepare('SELECT id, version, sha256, size FROM script_versions WHERE id = ?')
.get(id) as ScriptVersionMetadata | undefined;
}
// Latest version is 'max(version)' — used everywhere we want the freshest
// upload, whether or not it has been promoted (current_version_id).
export function getLatestVersion(scriptId: number): ScriptVersionRow | undefined {
return db()
.prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC LIMIT 1`)
.prepare(
`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC LIMIT 1`
)
.get(scriptId) as ScriptVersionRow | undefined;
}
@@ -191,7 +214,10 @@ export interface CreateScriptInput {
// Insert script + version 1 in one transaction. Scope may be set at creation
// because the script has no current_version_id yet (nothing to gate).
export function createScript(input: CreateScriptInput): { script: ScriptRow; version: ScriptVersionRow } {
export function createScript(input: CreateScriptInput): {
script: ScriptRow;
version: ScriptVersionRow;
} {
if (!input.name.trim()) throw new ScriptError('script name is required', 'name-empty');
const kind: ArtifactKind = input.kind ?? 'script';
if (!isArtifactKind(kind)) throw new ScriptError(`invalid kind: ${input.kind}`, 'invalid-kind');
@@ -200,15 +226,23 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
// the same on-disk cfg file, so the operator never picks it.
let targetRelPath = input.targetRelPath?.trim() ?? '';
if (!targetRelPath && (kind === 'config' || kind === 'baseline')) {
targetRelPath = configFileForGame(input.scopeType === 'game' ? input.scopeValue ?? null
: input.scopeValue?.startsWith('zm_') ? 't6' : 't5');
targetRelPath = configFileForGame(
input.scopeType === 'game'
? (input.scopeValue ?? null)
: input.scopeValue?.startsWith('zm_')
? 't6'
: 't5'
);
}
if (!targetRelPath) throw new ScriptError('target rel path is required', 'path-empty');
// Kind-specific validation.
if (kind === 'baseline') {
if (!input.scopeType || !input.scopeValue) {
throw new ScriptError('baselines require a scope (game or map)', 'baseline-requires-scope');
throw new ScriptError(
'baselines require a scope (game or map)',
'baseline-requires-scope'
);
}
if (getBaselineForScope(input.scopeType, input.scopeValue)) {
throw new ScriptError(
@@ -231,7 +265,8 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
}
}
validateScope(input.scopeType ?? null, input.scopeValue ?? null);
if (getScriptByName(input.name)) throw new ScriptError(`script "${input.name}" already exists`, 'name-taken');
if (getScriptByName(input.name))
throw new ScriptError(`script "${input.name}" already exists`, 'name-taken');
const sha = sha256(input.content);
const releaseVersion = normalizeReleaseVersion(input.releaseVersion);
@@ -249,9 +284,23 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
let scriptId!: number;
let versionId!: number;
db().transaction(() => {
const s = insertScript.run(input.name, targetRelPath, input.scopeType ?? null, input.scopeValue ?? null, kind);
const s = insertScript.run(
input.name,
targetRelPath,
input.scopeType ?? null,
input.scopeValue ?? null,
kind
);
scriptId = Number(s.lastInsertRowid);
const v = insertVersion.run(scriptId, sha, size, putFileBlob(input.content), input.notes ?? null, kind, releaseVersion);
const v = insertVersion.run(
scriptId,
sha,
size,
putFileBlob(input.content),
input.notes ?? null,
kind,
releaseVersion
);
versionId = Number(v.lastInsertRowid);
})();
@@ -261,7 +310,12 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
};
}
export function uploadNewVersion(scriptId: number, content: Buffer, notes?: string, releaseVersion?: string): ScriptVersionRow {
export function uploadNewVersion(
scriptId: number,
content: Buffer,
notes?: string,
releaseVersion?: string
): ScriptVersionRow {
const script = getScript(scriptId);
if (!script) throw new ScriptError(`no such script (id=${scriptId})`, 'no-such-script');
@@ -285,7 +339,16 @@ export function uploadNewVersion(scriptId: number, content: Buffer, notes?: stri
`INSERT INTO script_versions (script_id, version, sha256, size, content_hash, notes, kind, release_version)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(scriptId, nextVersion, sha, content.byteLength, putFileBlob(content), notes ?? null, script.kind, release);
.run(
scriptId,
nextVersion,
sha,
content.byteLength,
putFileBlob(content),
notes ?? null,
script.kind,
release
);
return getVersion(Number(res.lastInsertRowid))!;
}
@@ -293,7 +356,10 @@ function validateScope(scopeType: ScopeType | null, scopeValue: string | null):
if (scopeType === null && scopeValue === null) return;
if (scopeType === 'game' || scopeType === 'map') {
if (!scopeValue || !scopeValue.trim()) {
throw new ScriptError(`scope_value required for scope_type=${scopeType}`, 'invalid-scope');
throw new ScriptError(
`scope_value required for scope_type=${scopeType}`,
'invalid-scope'
);
}
return;
}
@@ -352,7 +418,11 @@ export function deleteVersion(versionId: number): void {
WHERE bs.script_version_id = ? AND b.state IN ('running','ready-promote') LIMIT 1`
)
.get(versionId) as { id: number } | undefined;
if (beta) throw new ScriptError(`beta ${beta.id} is still running against this version`, 'in-use-beta');
if (beta)
throw new ScriptError(
`beta ${beta.id} is still running against this version`,
'in-use-beta'
);
db().prepare('DELETE FROM script_versions WHERE id = ?').run(versionId);
}
@@ -370,7 +440,11 @@ export function deleteScript(scriptId: number): void {
WHERE bs.script_id = ? AND b.state IN ('running','ready-promote') LIMIT 1`
)
.get(scriptId) as { id: number } | undefined;
if (beta) throw new ScriptError(`beta ${beta.id} is still running against this script`, 'has-active-beta');
if (beta)
throw new ScriptError(
`beta ${beta.id} is still running against this script`,
'has-active-beta'
);
// Every FK that references scripts(id) or script_versions(id) with NO
// ACTION (the default) will abort the delete unless we null it first.
@@ -386,14 +460,37 @@ export function deleteScript(scriptId: number): void {
const handle = db();
const tx = handle.transaction((id: number) => {
const inMyVersions = 'IN (SELECT id FROM script_versions WHERE script_id = ?)';
const inMyBetas = 'IN (SELECT id FROM betas WHERE script_id = ? UNION SELECT beta_id FROM beta_scripts WHERE script_id = ?)';
const inMyBetas =
'IN (SELECT id FROM betas WHERE script_id = ? UNION SELECT beta_id FROM beta_scripts WHERE script_id = ?)';
handle.prepare(`UPDATE beta_servers SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE beta_server_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE deployment_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE deployments SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE deployments SET script_version_id = NULL WHERE script_version_id ${inMyVersions}`).run(id);
handle.prepare(`UPDATE deployments SET beta_id = NULL WHERE beta_id ${inMyBetas}`).run(id, id);
handle
.prepare(
`UPDATE beta_servers SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(
`UPDATE beta_server_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(
`UPDATE deployment_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(
`UPDATE deployments SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(
`UPDATE deployments SET script_version_id = NULL WHERE script_version_id ${inMyVersions}`
)
.run(id);
handle
.prepare(`UPDATE deployments SET beta_id = NULL WHERE beta_id ${inMyBetas}`)
.run(id, id);
handle.prepare('UPDATE deployments SET script_id = NULL WHERE script_id = ?').run(id);
handle.prepare('DELETE FROM scripts WHERE id = ?').run(id);
});
@@ -402,7 +499,10 @@ export function deleteScript(scriptId: number): void {
// Servers that match a script's scope. Empty for unscoped scripts — those are
// only ever pushed to explicitly chosen beta targets.
export function serversMatchingScope(scopeType: ScopeType | null, scopeValue: string | null): string[] {
export function serversMatchingScope(
scopeType: ScopeType | null,
scopeValue: string | null
): string[] {
if (scopeType === null || scopeValue === null) return [];
const rows = db()
.prepare(`SELECT name FROM servers WHERE ${scopeType} = ? ORDER BY name`)
@@ -415,7 +515,9 @@ export function serversMatchingScope(scopeType: ScopeType | null, scopeValue: st
// server.
export function listDistinctGames(): string[] {
const rows = db()
.prepare(`SELECT DISTINCT game FROM servers WHERE game IS NOT NULL AND game <> '' ORDER BY game`)
.prepare(
`SELECT DISTINCT game FROM servers WHERE game IS NOT NULL AND game <> '' ORDER BY game`
)
.all() as Array<{ game: string }>;
return rows.map((r) => r.game);
}
+35 -6
View File
@@ -7,13 +7,26 @@ import { applyRuntimePassword } from './runtime-config.js';
export const MAP_CONFIG_FILE = 'electric_cherry_map.cfg';
/** Loaded after Plutainer's default/main config and before map_rotate. */
export function getServerMapConfig(game: string | null, map: string | null, mapSlug?: string | null): Buffer | null {
export function getServerMapConfig(
game: string | null,
map: string | null,
mapSlug?: string | null
): Buffer | null {
const entry = selectedZombieMap(game, map, mapSlug);
return entry ? Buffer.from(`// Generated by Electric Cherry from the selected map.\nset sv_maprotation "${mapRotation(entry)}"\n`) : null;
return entry
? Buffer.from(
`// Generated by Electric Cherry from the selected map.\nset sv_maprotation "${mapRotation(entry)}"\n`
)
: null;
}
/** The promoted config used for first boot and drift reconciliation. */
export function getPromotedServerConfig(game: string | null, map: string | null, mapSlug?: string | null, rconPassword?: string | null): Buffer | null {
export function getPromotedServerConfig(
game: string | null,
map: string | null,
mapSlug?: string | null,
rconPassword?: string | null
): Buffer | null {
if (!game) return null;
const baseline = getBaselineForScope('game', game);
if (!baseline?.current_version_id) return null;
@@ -24,8 +37,24 @@ export function getPromotedServerConfig(game: string | null, map: string | null,
...listPromotedOverlaysForScope('game', game),
...(map ? listPromotedOverlaysForScope('map', map) : [])
].flatMap((overlay) => {
const current = overlay.current_version_id ? getVersion(overlay.current_version_id) : undefined;
return current ? [{ id: overlay.id, text: getVersionContent(current.id).toString('utf8') }] : [];
const current = overlay.current_version_id
? getVersion(overlay.current_version_id)
: undefined;
return current
? [{ id: overlay.id, text: getVersionContent(current.id).toString('utf8') }]
: [];
});
return Buffer.from(applyRuntimePassword(composeServerCfg(getVersionContent(version.id).toString('utf8'), overlays, game, map, mapSlug), rconPassword), 'utf8');
return Buffer.from(
applyRuntimePassword(
composeServerCfg(
getVersionContent(version.id).toString('utf8'),
overlays,
game,
map,
mapSlug
),
rconPassword
),
'utf8'
);
}
+118 -46
View File
@@ -15,7 +15,12 @@ function validGame(game: unknown): asserts game is KeyGame {
}
function normalizedKey(key: unknown): string {
if (typeof key !== 'string' || !key.trim() || key.trim().length > 512 || /\s/.test(key.trim())) {
if (
typeof key !== 'string' ||
!key.trim() ||
key.trim().length > 512 ||
/\s/.test(key.trim())
) {
throw new Error('Enter a server key without whitespace (maximum 512 characters)');
}
return key.trim();
@@ -29,22 +34,34 @@ function normalizedLabel(label: unknown): string {
}
export function listServerKeys(): ServerKey[] {
const rows = db().prepare('SELECT id,game,label,updated_at FROM server_keys ORDER BY game,label,id').all() as Omit<ServerKey, 'assigned_servers'>[];
const assignments = db().prepare(`SELECT DISTINCT k.id AS key_id,l.server_id FROM server_keys k
const rows = db()
.prepare('SELECT id,game,label,updated_at FROM server_keys ORDER BY game,label,id')
.all() as Omit<ServerKey, 'assigned_servers'>[];
const assignments = db()
.prepare(
`SELECT DISTINCT k.id AS key_id,l.server_id FROM server_keys k
JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key JOIN server_key_leases l ON l.key_id=occupied.id
ORDER BY l.server_id`).all() as { key_id: string; server_id: string }[];
ORDER BY l.server_id`
)
.all() as { key_id: string; server_id: string }[];
const byKey = new Map<string, string[]>();
for (const lease of assignments) {
const servers = byKey.get(lease.key_id) ?? [];
servers.push(lease.server_id);
byKey.set(lease.key_id, servers);
}
return rows.map(row => ({ ...row, assigned_servers: byKey.get(row.id) ?? [] }));
return rows.map((row) => ({ ...row, assigned_servers: byKey.get(row.id) ?? [] }));
}
export function getServerKey(serverId: string): string | undefined {
return (db().prepare(`SELECT k.pluto_key FROM server_key_leases l
JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`).get(serverId) as { pluto_key: string } | undefined)?.pluto_key;
return (
db()
.prepare(
`SELECT k.pluto_key FROM server_key_leases l
JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`
)
.get(serverId) as { pluto_key: string } | undefined
)?.pluto_key;
}
/** Saving a credential twice cannot increase available capacity. */
@@ -52,16 +69,27 @@ export function saveServerKey(game: unknown, key: unknown, label?: unknown): str
validGame(game);
const credential = normalizedKey(key);
const keyLabel = label === undefined ? undefined : normalizedLabel(label);
return db().transaction(() => {
const existing = db().prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?').get(game, credential) as { id: string } | undefined;
if (existing) {
if (keyLabel !== undefined) db().prepare("UPDATE server_keys SET label=?,updated_at=datetime('now') WHERE id=?").run(keyLabel, existing.id);
return existing.id;
}
const id = randomUUID();
db().prepare('INSERT INTO server_keys(id,game,label,pluto_key) VALUES(?,?,?,?)').run(id, game, keyLabel ?? '', credential);
return id;
}).immediate();
return db()
.transaction(() => {
const existing = db()
.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?')
.get(game, credential) as { id: string } | undefined;
if (existing) {
if (keyLabel !== undefined)
db()
.prepare(
"UPDATE server_keys SET label=?,updated_at=datetime('now') WHERE id=?"
)
.run(keyLabel, existing.id);
return existing.id;
}
const id = randomUUID();
db()
.prepare('INSERT INTO server_keys(id,game,label,pluto_key) VALUES(?,?,?,?)')
.run(id, game, keyLabel ?? '', credential);
return id;
})
.immediate();
}
// Compare the credential as well as its row: a mistakenly imported copy in
@@ -70,11 +98,18 @@ const availableKey = `NOT EXISTS (SELECT 1 FROM server_key_leases l
JOIN server_keys occupied ON occupied.id=l.key_id
WHERE occupied.pluto_key=k.pluto_key AND (l.server_id!=@server OR l.is_current=0))`;
export function serverKeyPool(game: KeyGame, serverId?: string): { total: number; available: number } {
export function serverKeyPool(
game: KeyGame,
serverId?: string
): { total: number; available: number } {
validGame(game);
return db().prepare(`SELECT COUNT(*) AS total,
return db()
.prepare(
`SELECT COUNT(*) AS total,
COALESCE(SUM(CASE WHEN ${availableKey} THEN 1 ELSE 0 END),0) AS available
FROM server_keys k WHERE k.game=@game`).get({ game, server: serverId ?? '' }) as { total: number; available: number };
FROM server_keys k WHERE k.game=@game`
)
.get({ game, server: serverId ?? '' }) as { total: number; available: number };
}
export function availableServerKeyCount(game: KeyGame): number {
@@ -85,38 +120,75 @@ export function availableServerKeyCount(game: KeyGame): number {
export function reserveServerKey(serverId: string, game: KeyGame, suppliedKey?: unknown): string {
validGame(game);
if (!serverId) throw new Error('Choose a server for the key reservation');
const credential = suppliedKey === undefined || suppliedKey === null || (typeof suppliedKey === 'string' && !suppliedKey.trim())
? undefined : normalizedKey(suppliedKey);
return db().transaction(() => {
const existing = db().prepare(`SELECT k.game,k.pluto_key FROM server_key_leases l
JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`).get(serverId) as { game: KeyGame; pluto_key: string } | undefined;
if (existing) {
if (existing.game !== game) throw new Error('This server already has a key reserved for another game');
if (credential && credential !== existing.pluto_key) throw new Error('This server already has a reserved key; destroy it before changing its key');
return existing.pluto_key;
}
const requestedId = credential ? saveServerKey(game, credential) : null;
const key = db().prepare(`SELECT k.id,k.pluto_key FROM server_keys k WHERE k.game=@game
const credential =
suppliedKey === undefined ||
suppliedKey === null ||
(typeof suppliedKey === 'string' && !suppliedKey.trim())
? undefined
: normalizedKey(suppliedKey);
return db()
.transaction(() => {
const existing = db()
.prepare(
`SELECT k.game,k.pluto_key FROM server_key_leases l
JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`
)
.get(serverId) as { game: KeyGame; pluto_key: string } | undefined;
if (existing) {
if (existing.game !== game)
throw new Error('This server already has a key reserved for another game');
if (credential && credential !== existing.pluto_key)
throw new Error(
'This server already has a reserved key; destroy it before changing its key'
);
return existing.pluto_key;
}
const requestedId = credential ? saveServerKey(game, credential) : null;
const key = db()
.prepare(
`SELECT k.id,k.pluto_key FROM server_keys k WHERE k.game=@game
AND (@requestedId IS NULL OR k.id=@requestedId) AND ${availableKey}
ORDER BY k.updated_at,k.id LIMIT 1`).get({ game, server: serverId, requestedId }) as { id: string; pluto_key: string } | undefined;
if (!key) throw new Error(credential ? 'This server key is already reserved by another server'
: `No available ${game.toUpperCase()} server keys. Add a key or destroy a server to release one.`);
db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(serverId, key.id);
return key.pluto_key;
}).immediate();
ORDER BY k.updated_at,k.id LIMIT 1`
)
.get({ game, server: serverId, requestedId }) as
{ id: string; pluto_key: string } | undefined;
if (!key)
throw new Error(
credential
? 'This server key is already reserved by another server'
: `No available ${game.toUpperCase()} server keys. Add a key or destroy a server to release one.`
);
db()
.prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)')
.run(serverId, key.id);
return key.pluto_key;
})
.immediate();
}
/** Only after confirmed destruction or a creation cancelled before any host mutation. */
export function releaseServerKey(serverId: string): boolean {
return db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(serverId).changes > 0;
return (
db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(serverId).changes > 0
);
}
export function removeServerKey(id: string): boolean {
return db().transaction(() => {
if (db().prepare(`SELECT 1 FROM server_keys k JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key
JOIN server_key_leases l ON l.key_id=occupied.id WHERE k.id=?`).get(id)) {
throw new Error('This key is reserved by a server. Destroy that server before removing its key.');
}
return db().prepare('DELETE FROM server_keys WHERE id=?').run(id).changes > 0;
}).immediate();
return db()
.transaction(() => {
if (
db()
.prepare(
`SELECT 1 FROM server_keys k JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key
JOIN server_key_leases l ON l.key_id=occupied.id WHERE k.id=?`
)
.get(id)
) {
throw new Error(
'This key is reserved by a server. Destroy that server before removing its key.'
);
}
return db().prepare('DELETE FROM server_keys WHERE id=?').run(id).changes > 0;
})
.immediate();
}
+56 -16
View File
@@ -35,8 +35,23 @@ export function matchServers(all: string[], patterns: string[]): string[] {
return [...all];
}
const regexes = patterns
.flatMap((p) => p.split(',').map((s) => s.trim()).filter(Boolean))
.map((p) => new RegExp('^' + p.replace(/[.+^$()|{}\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.') + '$'));
.flatMap((p) =>
p
.split(',')
.map((s) => s.trim())
.filter(Boolean)
)
.map(
(p) =>
new RegExp(
'^' +
p
.replace(/[.+^$()|{}\\]/g, '\\$&')
.replace(/\*/g, '.*')
.replace(/\?/g, '.') +
'$'
)
);
return all.filter((name) => regexes.some((r) => r.test(name)));
}
@@ -49,8 +64,13 @@ export async function ensureDest(
): Promise<{ ok: true; path: string } | { ok: false; reason: string }> {
const dest = destOf(server);
if (nativeInstance(server)) {
try { return (await managedStat(dest)).isDirectory() ? { ok: true, path: dest } : { ok: false, reason: 'Agent script directory is unavailable' }; }
catch (error) { return { ok: false, reason: (error as Error).message }; }
try {
return (await managedStat(dest)).isDirectory()
? { ok: true, path: dest }
: { ok: false, reason: 'Agent script directory is unavailable' };
} catch (error) {
return { ok: false, reason: (error as Error).message };
}
}
if (existsSync(dest)) {
try {
@@ -152,22 +172,34 @@ export async function seedServerRegistry(): Promise<number> {
}
export function getServer(name: string): ServerRow | undefined {
return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.name=?`).get(name) as ServerRow | undefined;
return db()
.prepare(
`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.name=?`
)
.get(name) as ServerRow | undefined;
}
export function listServers(): ServerRow[] {
return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.destroyed_at IS NULL ORDER BY s.name`).all() as ServerRow[];
return db()
.prepare(
`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.destroyed_at IS NULL ORDER BY s.name`
)
.all() as ServerRow[];
}
/** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */
export function activeBetaRemovalBlocker(server: string): string | null {
const betas = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
const betas = db()
.prepare(
`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')
ORDER BY b.id`).all(server) as { id: number }[];
ORDER BY b.id`
)
.all(server) as { id: number }[];
if (!betas.length) return null;
return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map(beta => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`;
return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map((beta) => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`;
}
export function setServerField(
@@ -178,7 +210,10 @@ export function setServerField(
setServerFields(name, { [field]: value });
}
export function setServerFields(name: string, fields: Partial<Record<keyof ServerRow, string | number | null>>): void {
export function setServerFields(
name: string,
fields: Partial<Record<keyof ServerRow, string | number | null>>
): void {
const allowed: Array<keyof ServerRow> = [
'container_state',
'container_checked_at',
@@ -211,10 +246,15 @@ export function setServerFields(name: string, fields: Partial<Record<keyof Serve
];
const entries = Object.entries(fields);
if (!entries.length) return;
for (const [field] of entries) if (!allowed.includes(field as keyof ServerRow)) throw new Error(`refusing to set ${field}`);
db().prepare(`UPDATE servers SET ${entries.map(([field]) => `${field} = ?`).join(', ')} WHERE name = ?
AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})`)
.run(...entries.map(([,value]) => value), name, ...entries.map(([,value]) => value));
for (const [field] of entries)
if (!allowed.includes(field as keyof ServerRow))
throw new Error(`refusing to set ${field}`);
db()
.prepare(
`UPDATE servers SET ${entries.map(([field]) => `${field} = ?`).join(', ')} WHERE name = ?
AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})`
)
.run(...entries.map(([, value]) => value), name, ...entries.map(([, value]) => value));
}
/**
+55 -14
View File
@@ -15,32 +15,62 @@ export function fleetSnapshot(serverName?: string) {
const hosts = listHosts();
const serverNow = Date.now();
const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host]));
const versions = new Map(runningVersions(serverName).map(version => [version.server, version]));
const instances = new Map((db().prepare(`SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
.all(...bindings) as NativeInstance[]).map(instance => [instance.server, instance]));
const versions = new Map(
runningVersions(serverName).map((version) => [version.server, version])
);
const instances = new Map(
(
db()
.prepare(
`SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`
)
.all(...bindings) as NativeInstance[]
).map((instance) => [instance.server, instance])
);
const holds = new Map<string, { target: string; betaId: number }[]>();
for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
for (const row of db()
.prepare(
`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id
JOIN beta_server_scripts bs ON bs.beta_id=b.id AND bs.server=bsv.server
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')
${serverName === undefined ? '' : 'AND bsv.server = ?'}`).all(...bindings) as
{ server: string; target: string; betaId: number }[]) {
const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries);
${serverName === undefined ? '' : 'AND bsv.server = ?'}`
)
.all(...bindings) as { server: string; target: string; betaId: number }[]) {
const entries = holds.get(row.server) ?? [];
entries.push({ target: row.target, betaId: row.betaId });
holds.set(row.server, entries);
}
const observe = serverName === undefined ? observationReader() : serverObservation;
const capacity = playerCapacityReader(serverName);
const selected = serverName === undefined ? undefined : getServer(serverName);
const rows = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
const rows =
serverName === undefined
? listServers()
: selected && !selected.destroyed_at
? [selected]
: [];
const servers = rows.map((s) => {
const instance = s.native_instance_id ? instances.get(s.name) : null;
return {
...observe(s),
...capacity(s.name),
host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
host: s.host_id ? (hostsById.get(s.host_id) ?? null) : null,
network: instance
? {
bindAddress: instance.bind_address,
localPort: instance.port,
publicAddress: instance.public_address,
publicPort: instance.public_port
}
: null,
plutonium: versions.get(s.name),
// The host agent owns the game connection.
endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
endpoint: instance
? { host: `agent:${instance.host_id}`, port: instance.port }
: s.game_port
? { host: s.rcon_host ?? rconHost, port: s.game_port }
: null,
held: holds.get(s.name) ?? []
};
});
@@ -49,11 +79,22 @@ export function fleetSnapshot(serverName?: string) {
serverNow,
// Include hosts without assigned servers. Re-evaluate heartbeat age on
// every existing snapshot refresh rather than adding a navigation poll.
hostAttention: hosts.filter(host => ['attention', 'offline'].includes(hostListStatus(host, serverNow).key)).length,
reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [],
hostAttention: hosts.filter((host) =>
['attention', 'offline'].includes(hostListStatus(host, serverNow).key)
).length,
reservedNames:
serverName === undefined
? (
db()
.prepare(
"SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')"
)
.all() as { server: string }[]
).map((row) => row.server)
: [],
integrations: {
native: true,
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
hostsReady: hosts.filter((host) => host.readiness === 'ready').length,
hostsTotal: hosts.length,
rcon: rconConfigured(),
rconTimeoutMs: config.rcon.timeoutMs,
+199 -59
View File
@@ -21,26 +21,38 @@ interface FileChange {
expectedBeforeSha?: string | null;
}
export interface StagedFileChange {
id: number; source_kind: SourceKind; source_id: string; server: string;
kind: ArtifactKind; rel_path: string; content: Buffer | null;
script_version_id: number | null; inventory_source: string;
deployment_file_id: number | null; preserve_before_image: number;
check_before: number; expected_before_sha: string | null;
state: 'pending' | 'applying' | 'applied' | 'cancelled'; applied_at: string | null;
id: number;
source_kind: SourceKind;
source_id: string;
server: string;
kind: ArtifactKind;
rel_path: string;
content: Buffer | null;
script_version_id: number | null;
inventory_source: string;
deployment_file_id: number | null;
preserve_before_image: number;
check_before: number;
expected_before_sha: string | null;
state: 'pending' | 'applying' | 'applied' | 'cancelled';
applied_at: string | null;
}
function targetPath(server: string, kind: ArtifactKind, relPath: string): string {
const root = resolve(destOfKind(server, kind));
const target = resolve(root, relPath);
const rel = relative(root, target);
if (!rel || rel === '..' || rel.startsWith('../') || isAbsolute(rel)) throw new Error('File target must be inside its artifact directory');
if (!rel || rel === '..' || rel.startsWith('../') || isAbsolute(rel))
throw new Error('File target must be inside its artifact directory');
return target;
}
/** Capture immutable desired bytes without changing the server or its inventory. */
export function queueFileChange(change: FileChange): void {
targetPath(change.server, change.kind, change.relPath);
db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content_hash,
db()
.prepare(
`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content_hash,
script_version_id, inventory_source, deployment_file_id, preserve_before_image, check_before, expected_before_sha)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(source_kind, source_id, server, kind, rel_path) DO UPDATE SET
@@ -48,51 +60,122 @@ export function queueFileChange(change: FileChange): void {
inventory_source = excluded.inventory_source, deployment_file_id = excluded.deployment_file_id,
preserve_before_image = excluded.preserve_before_image, check_before = excluded.check_before,
expected_before_sha = excluded.expected_before_sha
WHERE staged_file_changes.state = 'pending'`)
.run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, putFileBlob(change.content),
change.scriptVersionId ?? null, change.inventorySource, change.deploymentFileId ?? null,
change.preserveBeforeImage ? 1 : 0, change.expectedBeforeSha !== undefined ? 1 : 0, change.expectedBeforeSha ?? null);
WHERE staged_file_changes.state = 'pending'`
)
.run(
change.sourceKind,
change.sourceId,
change.server,
change.kind,
change.relPath,
putFileBlob(change.content),
change.scriptVersionId ?? null,
change.inventorySource,
change.deploymentFileId ?? null,
change.preserveBeforeImage ? 1 : 0,
change.expectedBeforeSha !== undefined ? 1 : 0,
change.expectedBeforeSha ?? null
);
}
export function stagedFileChanges(sourceKind: SourceKind, sourceId: string, server?: string): StagedFileChange[] {
return db().prepare(`SELECT *, ${blobContent('content_hash')} AS content FROM staged_file_changes WHERE source_kind = ? AND source_id = ?
${server === undefined ? '' : 'AND server = ?'} ORDER BY id`)
export function stagedFileChanges(
sourceKind: SourceKind,
sourceId: string,
server?: string
): StagedFileChange[] {
return db()
.prepare(
`SELECT *, ${blobContent('content_hash')} AS content FROM staged_file_changes WHERE source_kind = ? AND source_id = ?
${server === undefined ? '' : 'AND server = ?'} ORDER BY id`
)
.all(sourceKind, sourceId, ...(server === undefined ? [] : [server])) as StagedFileChange[];
}
export function cancelStagedFiles(sourceKind: SourceKind, sourceId: string, server?: string): void {
db().prepare(`UPDATE staged_file_changes SET state = 'cancelled' WHERE source_kind = ? AND source_id = ?
AND state = 'pending' ${server === undefined ? '' : 'AND server = ?'}`)
db()
.prepare(
`UPDATE staged_file_changes SET state = 'cancelled' WHERE source_kind = ? AND source_id = ?
AND state = 'pending' ${server === undefined ? '' : 'AND server = ?'}`
)
.run(sourceKind, sourceId, ...(server === undefined ? [] : [server]));
}
export function hasAppliedFiles(sourceKind: SourceKind, sourceId: string, server?: string): boolean {
return stagedFileChanges(sourceKind, sourceId, server).some(row => row.state === 'applied' || row.state === 'applying');
export function hasAppliedFiles(
sourceKind: SourceKind,
sourceId: string,
server?: string
): boolean {
return stagedFileChanges(sourceKind, sourceId, server).some(
(row) => row.state === 'applied' || row.state === 'applying'
);
}
/** null identifies legacy work that wrote files before staging was introduced. */
export function isFileApplied(sourceKind: SourceKind, sourceId: string, server: string, kind: ArtifactKind, relPath: string): boolean | null {
const row = stagedFileChanges(sourceKind, sourceId, server).find(row => row.kind === kind && row.rel_path === relPath);
export function isFileApplied(
sourceKind: SourceKind,
sourceId: string,
server: string,
kind: ArtifactKind,
relPath: string
): boolean | null {
const row = stagedFileChanges(sourceKind, sourceId, server).find(
(row) => row.kind === kind && row.rel_path === relPath
);
return row ? row.state === 'applied' || row.state === 'applying' : null;
}
function readExisting(path: string): Buffer | null {
try { return readFileSync(path); }
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err; }
try {
return readFileSync(path);
} catch (err) {
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw err;
}
}
/** The first activation owns the rollback image, including an absent file. */
export function captureBetaBeforeImage(row: Pick<StagedFileChange, 'source_id' | 'server' | 'rel_path' | 'kind'>, before: Buffer | null): void {
if (db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?')
.get(row.source_id, row.server, row.rel_path)) return;
export function captureBetaBeforeImage(
row: Pick<StagedFileChange, 'source_id' | 'server' | 'rel_path' | 'kind'>,
before: Buffer | null
): void {
if (
db()
.prepare(
'SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?'
)
.get(row.source_id, row.server, row.rel_path)
)
return;
const prior = getServerScript(row.server, row.rel_path);
if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots
(beta_id, server, rel_path, prior_content_hash, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
.run(row.source_id, row.server, row.rel_path, putFileBlob(before), sha256Buffer(before), before.length);
db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`)
.run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, row.server, row.rel_path, row.kind);
db().prepare('INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
if (before !== null)
db()
.prepare(
`INSERT OR IGNORE INTO beta_file_snapshots
(beta_id, server, rel_path, prior_content_hash, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`
)
.run(
row.source_id,
row.server,
row.rel_path,
putFileBlob(before),
sha256Buffer(before),
before.length
);
db()
.prepare(
`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`
)
.run(
before === null ? 0 : 1,
prior?.script_version_id ?? null,
row.source_id,
row.server,
row.rel_path,
row.kind
);
db()
.prepare('INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
.run(row.source_id, row.server, row.rel_path);
}
@@ -101,25 +184,44 @@ export function captureBetaBeforeImage(row: Pick<StagedFileChange, 'source_id' |
* from interleaving between the final gates and dispatching the restart.
*/
export function applyStagedFilesForOperation(operationId: string, server: string): void {
const rows = db().prepare(`SELECT f.*, ${blobContent('f.content_hash')} AS content FROM staged_file_changes f JOIN provisioning_sources s
const rows = db()
.prepare(
`SELECT f.*, ${blobContent('f.content_hash')} AS content FROM staged_file_changes f JOIN provisioning_sources s
ON s.kind = f.source_kind AND s.source_id = f.source_id AND s.server = f.server
WHERE s.operation_id = ? AND f.server = ? AND f.state IN ('pending','applying') ORDER BY f.id`)
WHERE s.operation_id = ? AND f.server = ? AND f.state IN ('pending','applying') ORDER BY f.id`
)
.all(operationId, server) as StagedFileChange[];
if (!rows.length) return;
if (!statSync(serverDir(server)).isDirectory()) throw new Error('Server directory is unavailable');
if (!statSync(serverDir(server)).isDirectory())
throw new Error('Server directory is unavailable');
// Preflight every target before changing any files. An interrupted write
// needs inspection/rollback, never an automatic replay.
const plans = rows.map(row => {
if (row.state === 'applying') throw new Error('A previous file write was interrupted; inspect or roll back the campaign');
const source = row.source_kind === 'campaign'
? db().prepare(`SELECT 1 FROM deployment_servers WHERE deployment_id = ? AND server = ?
AND server_deleted_at IS NULL AND provisioning_id = ? AND restart_state IN ('pending','failed')`).get(row.source_id, server, operationId)
: db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.beta_id = ? AND bs.server = ?
AND bs.server_deleted_at IS NULL AND bs.provisioning_id = ? AND b.state IN ('running','ready-promote')`).get(row.source_id, server, operationId);
const plans = rows.map((row) => {
if (row.state === 'applying')
throw new Error(
'A previous file write was interrupted; inspect or roll back the campaign'
);
const source =
row.source_kind === 'campaign'
? db()
.prepare(
`SELECT 1 FROM deployment_servers WHERE deployment_id = ? AND server = ?
AND server_deleted_at IS NULL AND provisioning_id = ? AND restart_state IN ('pending','failed')`
)
.get(row.source_id, server, operationId)
: db()
.prepare(
`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.beta_id = ? AND bs.server = ?
AND bs.server_deleted_at IS NULL AND bs.provisioning_id = ? AND b.state IN ('running','ready-promote')`
)
.get(row.source_id, server, operationId);
if (!source) throw new Error('File change requester is no longer eligible');
const path = targetPath(server, row.kind, row.rel_path);
const before = readExisting(path);
if (row.check_before && (before === null ? null : sha256Buffer(before)) !== row.expected_before_sha) {
if (
row.check_before &&
(before === null ? null : sha256Buffer(before)) !== row.expected_before_sha
) {
throw new Error(`${row.rel_path}: changed while queued; scan and review again`);
}
return { row, path, before };
@@ -132,14 +234,30 @@ export function applyStagedFilesForOperation(operationId: string, server: string
captureBetaBeforeImage(row, before);
}
if (row.deployment_file_id && !row.preserve_before_image) {
db().prepare(`UPDATE deployment_files SET
verb = ?, content_before_hash = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`)
.run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', putFileBlob(before),
before === null ? null : sha256Buffer(before), row.deployment_file_id);
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${server}: captured rollback snapshot for ${row.rel_path}`);
db()
.prepare(
`UPDATE deployment_files SET
verb = ?, content_before_hash = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`
)
.run(
row.content === null ? 'removed' : before === null ? 'added' : 'updated',
putFileBlob(before),
before === null ? null : sha256Buffer(before),
row.deployment_file_id
);
if (row.source_kind === 'campaign')
db()
.prepare(
'INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)'
)
.run(
row.source_id,
`${server}: captured rollback snapshot for ${row.rel_path}`
);
}
db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?").run(row.id);
db()
.prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?")
.run(row.id);
})();
if (row.content === null) {
rmSync(path, { force: true });
@@ -151,17 +269,39 @@ export function applyStagedFilesForOperation(operationId: string, server: string
writeFileSync(path, row.content);
// A queued version can be deleted independently; the staged bytes
// still apply, but its optional inventory link must remain valid.
const version = row.script_version_id !== null && db().prepare('SELECT 1 FROM script_versions WHERE id = ?').get(row.script_version_id)
? row.script_version_id : null;
recordScript(server, row.rel_path, version, sha256Buffer(row.content), row.content.length, row.inventory_source);
const version =
row.script_version_id !== null &&
db()
.prepare('SELECT 1 FROM script_versions WHERE id = ?')
.get(row.script_version_id)
? row.script_version_id
: null;
recordScript(
server,
row.rel_path,
version,
sha256Buffer(row.content),
row.content.length,
row.inventory_source
);
}
db().prepare("UPDATE staged_file_changes SET state = 'applied', applied_at = ? WHERE id = ?")
db()
.prepare(
"UPDATE staged_file_changes SET state = 'applied', applied_at = ? WHERE id = ?"
)
.run(new Date().toISOString(), row.id);
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${server}: applied ${row.rel_path}`);
if (row.source_kind === 'campaign')
db()
.prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${server}: applied ${row.rel_path}`);
}
for (const betaId of new Set(rows.filter(row => row.source_kind === 'beta').map(row => row.source_id))) {
db().prepare('UPDATE beta_servers SET deployed_at = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL')
for (const betaId of new Set(
rows.filter((row) => row.source_kind === 'beta').map((row) => row.source_id)
)) {
db()
.prepare(
'UPDATE beta_servers SET deployed_at = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL'
)
.run(new Date().toISOString(), new Date().toISOString(), betaId, server);
}
}
+68 -22
View File
@@ -1,32 +1,49 @@
import { db } from '../db/index.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { rconGetPlayerCount, rconStatusResult, type RconStatusResult } from '../integrations/rcon.js';
import {
rconGetPlayerCount,
rconStatusResult,
type RconStatusResult
} from '../integrations/rcon.js';
import { latestGameSubmission } from './lifecycle-evidence.js';
export { latestGameSubmission } from './lifecycle-evidence.js';
export interface TelemetryEvidence { players: number; map: string; receivedAt: string }
export interface TelemetryEvidence {
players: number;
map: string;
receivedAt: string;
}
export function telemetryReport(server: string): GameTelemetry | null {
const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as
{ received_at: string; payload: string | null } | undefined;
const row = db()
.prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?')
.get(server) as { received_at: string; payload: string | null } | undefined;
if (!row?.payload) return null;
try {
const saved = JSON.parse(row.payload) as GameTelemetry;
const event = parseGameEvent(saved);
return event.server === server ? { ...event, receivedAt: row.received_at, timing: saved.timing } : null;
} catch { return null; }
return event.server === server
? { ...event, receivedAt: row.received_at, timing: saved.timing }
: null;
} catch {
return null;
}
}
/** Only validated, recently received reports can establish current game state. */
export function telemetryEvidence(server: string, after: string | null = latestGameSubmission(server)): TelemetryEvidence | null {
export function telemetryEvidence(
server: string,
after: string | null = latestGameSubmission(server)
): TelemetryEvidence | null {
const report = telemetryReport(server);
if (!report) return null;
const received = Date.parse(report.receivedAt);
const age = Date.now() - received;
if (!Number.isFinite(age) || age < 0 || age > ONLINE_MAX_MS) return null;
if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after))) return null;
if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after)))
return null;
return { players: report.players.length, map: report.map, receivedAt: report.receivedAt };
}
@@ -45,39 +62,68 @@ export interface PlayerEvidence {
/** The same eligibility decision, with provenance from the probes already performed. */
export async function probePlayerEvidence(server: string): Promise<PlayerEvidence> {
const fromTelemetry = (value: TelemetryEvidence): PlayerEvidence => ({
players: value.players, source: 'telemetry', observedAt: new Date().toISOString(), receivedAt: value.receivedAt
players: value.players,
source: 'telemetry',
observedAt: new Date().toISOString(),
receivedAt: value.receivedAt
});
const occupied = telemetryEvidence(server);
if (occupied && occupied.players > 0) return fromTelemetry(occupied);
const players = await rconGetPlayerCount(server).catch(() => null);
// Read after the network request: a player may join while RCON is pending.
const telemetry = telemetryEvidence(server);
if (players !== null && players > 0) return { players, source: 'rcon', observedAt: new Date().toISOString() };
if (players !== null && players > 0)
return { players, source: 'rcon', observedAt: new Date().toISOString() };
if (telemetry && telemetry.players > 0) return fromTelemetry(telemetry);
if (players === 0) return { players, source: 'rcon', observedAt: new Date().toISOString() };
if (telemetry?.players === 0) return fromTelemetry(telemetry);
return { players: null, source: null, observedAt: new Date().toISOString() };
}
export type VerifiedGameStatus = { ok: true; players: number; map: string | null; observedAt: string } & (
{ source: 'rcon'; rcon: Extract<RconStatusResult, { ok: true }> }
| { source: 'telemetry'; receivedAt: string });
export type VerifiedGameStatus = {
ok: true;
players: number;
map: string | null;
observedAt: string;
} & (
| { source: 'rcon'; rcon: Extract<RconStatusResult, { ok: true }> }
| { source: 'telemetry'; receivedAt: string }
);
/** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */
export async function probeServerStatus(server: string, after?: string | null): Promise<
VerifiedGameStatus | { ok: false; reason: string; observedAt: string }
> {
export async function probeServerStatus(
server: string,
after?: string | null
): Promise<VerifiedGameStatus | { ok: false; reason: string; observedAt: string }> {
const current = telemetryEvidence(server, after);
if (current) return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() };
let reason = 'RCON unavailable';
if (current)
return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() };
let reason: string;
try {
const rcon = await rconStatusResult(server);
const latest = telemetryEvidence(server, after);
if (latest) return { ok: true, ...latest, source: 'telemetry', observedAt: new Date().toISOString() };
if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', rcon, observedAt: new Date().toISOString() };
if (latest)
return {
ok: true,
...latest,
source: 'telemetry',
observedAt: new Date().toISOString()
};
if (rcon.ok)
return {
ok: true,
players: rcon.status.players,
map: rcon.status.map,
source: 'rcon',
rcon,
observedAt: new Date().toISOString()
};
reason = rcon.reason;
} catch (err) { reason = (err as Error).message; }
} catch (err) {
reason = (err as Error).message;
}
const telemetry = telemetryEvidence(server, after);
return telemetry ? { ok: true, ...telemetry, source: 'telemetry', observedAt: new Date().toISOString() }
return telemetry
? { ok: true, ...telemetry, source: 'telemetry', observedAt: new Date().toISOString() }
: { ok: false, reason, observedAt: new Date().toISOString() };
}
+19 -7
View File
@@ -3,12 +3,20 @@ import type { ServerObservation } from '../../server-observation.js';
import type { ServerRow } from './servers.js';
/** Monitoring and provisioning publish the same complete RCON observation. */
export function successfulStatusFields(result: Extract<RconStatusResult, { ok: true }>, observedAt: string): Partial<ServerRow> {
export function successfulStatusFields(
result: Extract<RconStatusResult, { ok: true }>,
observedAt: string
): Partial<ServerRow> {
return {
last_checked_at: observedAt, last_seen_at: observedAt,
last_players: result.status.players, last_map: result.status.map,
last_gametype: result.status.gametype, last_roster: JSON.stringify(result.status.roster),
last_reply_snippet: null, last_error: null, rcon_failures: 0
last_checked_at: observedAt,
last_seen_at: observedAt,
last_players: result.status.players,
last_map: result.status.map,
last_gametype: result.status.gametype,
last_roster: JSON.stringify(result.status.roster),
last_reply_snippet: null,
last_error: null,
rcon_failures: 0
};
}
@@ -17,6 +25,10 @@ export function activityFields(current: ServerRow, status: ServerObservation): P
if (status.health !== 'online' || status.last_players === null) return {};
return status.last_players > 0
? { active_since: current.active_since ?? status.last_seen_at, inactive_since: null }
: { active_since: null, inactive_since: current.inactive_since ?? status.last_seen_at,
game_started_at: null, game_ended_at: null };
: {
active_since: null,
inactive_since: current.inactive_since ?? status.last_seen_at,
game_started_at: null,
game_ended_at: null
};
}
+96 -28
View File
@@ -1,4 +1,8 @@
import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js';
import {
rconHealthError,
selectServerObservation,
type GameObservation
} from '../../server-observation.js';
import { classifyHealth } from './health.js';
import { isProvisioning } from './provisioning.js';
import { restartSettling, recoverySettling } from '../provisioning/restarts.js';
@@ -8,32 +12,71 @@ import { latestGameSubmissions } from './lifecycle-evidence.js';
import { db } from '../db/index.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null }
interface Evidence {
provisioning: boolean;
settling: boolean;
telemetry: GameTelemetry | null;
submission: string | null;
}
/** One set of evidence reads per snapshot, rather than four queries per server. */
export function observationReader() {
const handle = db();
const operations = new Map((handle.prepare(`SELECT id, server, action, state FROM server_deployments
WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as
{ id: string; server: string; action: string; state: string }[]).map(row => [row.server, row]));
const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout').all() as
{ server: string; requested_at: string; healthy_at: string | null; operation_id: string | null }[]).map(row => [row.server, row]));
const operations = new Map(
(
handle
.prepare(
`SELECT id, server, action, state FROM server_deployments
WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`
)
.all() as { id: string; server: string; action: string; state: string }[]
).map((row) => [row.server, row])
);
const restarts = new Map(
(
handle
.prepare(
'SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout'
)
.all() as {
server: string;
requested_at: string;
healthy_at: string | null;
operation_id: string | null;
}[]
).map((row) => [row.server, row])
);
const submissions = latestGameSubmissions();
const reports = new Map<string, GameTelemetry>();
for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
{ server: string; received_at: string; payload: string }[]) {
for (const row of handle
.prepare(
'SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL'
)
.all() as { server: string; received_at: string; payload: string }[]) {
try {
const saved = JSON.parse(row.payload) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.server) reports.set(row.server, { ...event, receivedAt: row.received_at, timing: saved.timing });
} catch { /* Invalid retained telemetry is unavailable. */ }
if (event.server === row.server)
reports.set(row.server, {
...event,
receivedAt: row.received_at,
timing: saved.timing
});
} catch {
/* Invalid retained telemetry is unavailable. */
}
}
return (server: ServerRow) => {
const operation = operations.get(server.name), restart = restarts.get(server.name);
const operation = operations.get(server.name),
restart = restarts.get(server.name);
return serverObservation(server, {
provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
provisioning:
!!operation &&
['create', 'rebuild', 'start'].includes(operation.action) &&
['queued', 'running', 'awaiting-health'].includes(operation.state),
settling: recoverySettling(restart, operation),
telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
telemetry: reports.get(server.name) ?? null,
submission: submissions.get(server.name) ?? null
});
};
}
@@ -41,22 +84,47 @@ export function observationReader() {
/** Shared snapshot for REST, status events, metrics, and other server-side readers. */
export function serverObservation(server: ServerRow, evidence?: Evidence) {
let roster = null;
try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error,
rcon_failures: server.rcon_failures, roster };
try {
roster = server.last_roster ? JSON.parse(server.last_roster) : null;
} catch {
/* unavailable */
}
const rcon: GameObservation = {
last_players: server.last_players,
last_map: server.last_map,
last_gametype: server.last_gametype,
last_seen_at: server.last_seen_at,
last_error: server.last_error,
rcon_failures: server.rcon_failures,
roster
};
const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
const state = (evidence ? evidence.settling : restartSettling(server.name))
? 'restarting'
: server.container_state;
const ready = !provisioning && (!state || state === 'running' || state === 'unknown');
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, rconHealthError(rcon), state);
const health = provisioning
? 'provisioning'
: classifyHealth(server.last_seen_at, rconHealthError(rcon), state);
// This projection is shared by viewer-visible snapshots. Never spread saved diagnostics.
const { last_reply_snippet: _snippet, ...publicServer } = server;
return selectServerObservation({ ...publicServer, ...rcon, rcon_status: rcon, container_state: state, health,
status_source: null as 'telemetry' | 'rcon' | null,
telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,
telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name),
last_error: ready ? server.last_error : null,
active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null,
game_started_at: ready ? server.game_started_at : null, game_ended_at: ready ? server.game_ended_at : null
}, undefined, Date.now());
return selectServerObservation(
{
...publicServer,
...rcon,
rcon_status: rcon,
container_state: state,
health,
status_source: null as 'telemetry' | 'rcon' | null,
telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,
telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name),
last_error: ready ? server.last_error : null,
active_since: ready ? server.active_since : null,
inactive_since: ready ? server.inactive_since : null,
game_started_at: ready ? server.game_started_at : null,
game_ended_at: ready ? server.game_ended_at : null
},
undefined,
Date.now()
);
}
+53 -20
View File
@@ -6,11 +6,18 @@ const MAX_PAYLOAD_LENGTH = 16 * 1024;
// Bound storage even when requests claim many different, unregistered names.
export function telemetryRetention() {
const settings = fleetSettings();
return { maxAgeDays: settings.telemetryKeepDays, maxPerServer: settings.telemetryMaxPerServer, maxTotal: settings.telemetryMaxTotal };
return {
maxAgeDays: settings.telemetryKeepDays,
maxPerServer: settings.telemetryMaxPerServer,
maxTotal: settings.telemetryMaxTotal
};
}
export const telemetryCutoff = () => telemetryRetention().maxAgeDays ? new Date(Date.now() - telemetryRetention().maxAgeDays * 86_400_000).toISOString() : '';
export const telemetryCutoff = () =>
telemetryRetention().maxAgeDays
? new Date(Date.now() - telemetryRetention().maxAgeDays * 86_400_000).toISOString()
: '';
const cutoff = telemetryCutoff;
const textField = (value: unknown) => typeof value === 'string' ? value.slice(0, 255) : null;
const textField = (value: unknown) => (typeof value === 'string' ? value.slice(0, 255) : null);
export function recordTelemetryRequest(input: {
value: unknown;
@@ -27,29 +34,55 @@ export function recordTelemetryRequest(input: {
const payload = JSON.stringify(body);
const handle = db();
handle.transaction(() => {
handle.prepare(`INSERT INTO telemetry_requests
handle
.prepare(
`INSERT INTO telemetry_requests
(server, received_at, status, error, event, map, round, player_count, content_type, payload, payload_truncated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
body.server, input.receivedAt, input.status, input.error?.slice(0, 2000) ?? null,
textField(body.event), textField(body.map),
Number.isSafeInteger(body.round) ? body.round as number : null,
Array.isArray(body.players) ? body.players.length : null,
textField(input.contentType), payload.slice(0, MAX_PAYLOAD_LENGTH), Number(payload.length > MAX_PAYLOAD_LENGTH)
);
if (telemetryRetention().maxAgeDays) handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff());
if (telemetryRetention().maxPerServer) handle.prepare(`DELETE FROM telemetry_requests WHERE server = ? AND id <= (
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(
body.server,
input.receivedAt,
input.status,
input.error?.slice(0, 2000) ?? null,
textField(body.event),
textField(body.map),
Number.isSafeInteger(body.round) ? (body.round as number) : null,
Array.isArray(body.players) ? body.players.length : null,
textField(input.contentType),
payload.slice(0, MAX_PAYLOAD_LENGTH),
Number(payload.length > MAX_PAYLOAD_LENGTH)
);
if (telemetryRetention().maxAgeDays)
handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff());
if (telemetryRetention().maxPerServer)
handle
.prepare(
`DELETE FROM telemetry_requests WHERE server = ? AND id <= (
SELECT id FROM telemetry_requests WHERE server = ? ORDER BY id DESC LIMIT 1 OFFSET ?
)`).run(body.server, body.server, telemetryRetention().maxPerServer);
if (telemetryRetention().maxTotal) handle.prepare(`DELETE FROM telemetry_requests WHERE id <= (
)`
)
.run(body.server, body.server, telemetryRetention().maxPerServer);
if (telemetryRetention().maxTotal)
handle
.prepare(
`DELETE FROM telemetry_requests WHERE id <= (
SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 1 OFFSET ?
)`).run(telemetryRetention().maxTotal);
)`
)
.run(telemetryRetention().maxTotal);
})();
}
export function listTelemetryRequests(server: string): TelemetryRequestLogEntry[] {
const rows = db().prepare(`SELECT id, server, received_at AS receivedAt, status, error, event, map, round,
const rows = db()
.prepare(
`SELECT id, server, received_at AS receivedAt, status, error, event, map, round,
player_count AS playerCount, content_type AS contentType, payload, payload_truncated AS payloadTruncated
FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`)
.all(server, cutoff(), telemetryRetention().maxPerServer || 10000) as Array<Omit<TelemetryRequestLogEntry, 'payloadTruncated'> & { payloadTruncated: number }>;
return rows.map(row => ({ ...row, payloadTruncated: !!row.payloadTruncated }));
FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`
)
.all(server, cutoff(), telemetryRetention().maxPerServer || 10000) as Array<
Omit<TelemetryRequestLogEntry, 'payloadTruncated'> & { payloadTruncated: number }
>;
return rows.map((row) => ({ ...row, payloadTruncated: !!row.payloadTruncated }));
}
+89 -28
View File
@@ -5,58 +5,119 @@ import { recordLeaderboardTelemetry } from './leaderboard-ingest.js';
import { latestGameSubmission } from './status-evidence.js';
export function listTelemetry(): GameTelemetry[] {
const rows = db().prepare(`SELECT t.payload FROM server_telemetry t JOIN servers s ON s.name = t.server
WHERE s.destroyed_at IS NULL AND t.payload IS NOT NULL`).all() as Array<{ payload: string }>;
return rows.map(row => JSON.parse(row.payload) as GameTelemetry);
const rows = db()
.prepare(
`SELECT t.payload FROM server_telemetry t JOIN servers s ON s.name = t.server
WHERE s.destroyed_at IS NULL AND t.payload IS NOT NULL`
)
.all() as Array<{ payload: string }>;
return rows.map((row) => JSON.parse(row.payload) as GameTelemetry);
}
export function saveTelemetry(report: GameTelemetry): GameTelemetry {
// Keep callers outside the HTTP route atomic too.
const handle = db();
return handle.transaction(() => {
const row = handle.prepare('SELECT payload FROM server_telemetry WHERE server = ?').get(report.server) as { payload: string | null } | undefined;
const previous: GameTelemetry | undefined = row?.payload ? JSON.parse(row.payload) : undefined;
const row = handle
.prepare('SELECT payload FROM server_telemetry WHERE server = ?')
.get(report.server) as { payload: string | null } | undefined;
const previous: GameTelemetry | undefined = row?.payload
? JSON.parse(row.payload)
: undefined;
recordLeaderboardTelemetry(handle, report);
// Record each occupancy transition at receipt, before a heartbeat can
// replace game_start or an empty report between status polls.
const after = latestGameSubmission(report.server);
if (!after || Date.parse(report.receivedAt) > Date.parse(after)) {
if (report.players.length > 0) {
handle.prepare('UPDATE servers SET active_since = COALESCE(active_since, ?), inactive_since = NULL WHERE name = ?')
handle
.prepare(
'UPDATE servers SET active_since = COALESCE(active_since, ?), inactive_since = NULL WHERE name = ?'
)
.run(report.receivedAt, report.server);
if (report.event === 'game_start') {
handle.prepare('UPDATE servers SET game_started_at = ?, game_ended_at = NULL WHERE name = ?').run(report.receivedAt, report.server);
handle
.prepare(
'UPDATE servers SET game_started_at = ?, game_ended_at = NULL WHERE name = ?'
)
.run(report.receivedAt, report.server);
} else if (report.event === 'game_end') {
handle.prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?').run(report.receivedAt, report.server);
handle
.prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?')
.run(report.receivedAt, report.server);
}
} else {
handle.prepare(`UPDATE servers SET active_since = NULL, inactive_since = COALESCE(inactive_since, ?),
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(report.receivedAt, report.server);
handle
.prepare(
`UPDATE servers SET active_since = NULL, inactive_since = COALESCE(inactive_since, ?),
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`
)
.run(report.receivedAt, report.server);
}
}
const serverTiming = handle.prepare(`SELECT active_since AS activeSince, inactive_since AS inactiveSince,
game_started_at AS gameStartedAt, game_ended_at AS gameEndedAt FROM servers WHERE name = ?`).get(report.server) as ServerTiming;
const match = handle.prepare('SELECT id FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1').get(report.server) as { id: number };
const serverTiming = handle
.prepare(
`SELECT active_since AS activeSince, inactive_since AS inactiveSince,
game_started_at AS gameStartedAt, game_ended_at AS gameEndedAt FROM servers WHERE name = ?`
)
.get(report.server) as ServerTiming;
const match = handle
.prepare('SELECT id FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1')
.get(report.server) as { id: number };
const sameMatch = previous?.timing?.matchId === match.id;
// A gap beyond the normal heartbeat window cannot establish continuity.
const continuous = previous && Date.parse(report.receivedAt) - Date.parse(previous.receivedAt) <= 120_000;
report = { ...report, timing: { matchId: match.id, server: serverTiming, players: Object.fromEntries(report.players.map(player => {
const prior = continuous ? previous?.timing?.players[player.guid] : undefined;
return [player.guid, {
serverJoinedAt: prior?.serverJoinedAt ?? report.receivedAt,
matchJoinedAt: sameMatch && prior ? prior.matchJoinedAt : report.receivedAt,
roundStarted: sameMatch && prior ? prior.roundStarted : report.event === 'game_start' ? 1 : report.round
}];
})) } };
handle.prepare(`INSERT INTO server_telemetry (server, round, players, received_at, payload)
const continuous =
previous && Date.parse(report.receivedAt) - Date.parse(previous.receivedAt) <= 120_000;
report = {
...report,
timing: {
matchId: match.id,
server: serverTiming,
players: Object.fromEntries(
report.players.map((player) => {
const prior = continuous
? previous?.timing?.players[player.guid]
: undefined;
return [
player.guid,
{
serverJoinedAt: prior?.serverJoinedAt ?? report.receivedAt,
matchJoinedAt:
sameMatch && prior ? prior.matchJoinedAt : report.receivedAt,
roundStarted:
sameMatch && prior
? prior.roundStarted
: report.event === 'game_start'
? 1
: report.round
}
];
})
)
}
};
handle
.prepare(
`INSERT INTO server_telemetry (server, round, players, received_at, payload)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(server) DO UPDATE SET
round = excluded.round, players = excluded.players, received_at = excluded.received_at, payload = excluded.payload`)
.run(report.server, report.event === 'game_start' ? 0 : report.round, report.players.length, report.receivedAt, JSON.stringify(report));
round = excluded.round, players = excluded.players, received_at = excluded.received_at, payload = excluded.payload`
)
.run(
report.server,
report.event === 'game_start' ? 0 : report.round,
report.players.length,
report.receivedAt,
JSON.stringify(report)
);
if (report.event === 'game_end') {
// Legacy betas also take match completion directly from the script.
if (report.round >= MIN_RECORDED_ROUND) handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND server_deleted_at IS NULL AND stage = 'awaiting-match'
AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`)
.run(report.receivedAt, report.server, report.receivedAt);
if (report.round >= MIN_RECORDED_ROUND)
handle
.prepare(
`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND server_deleted_at IS NULL AND stage = 'awaiting-match'
AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`
)
.run(report.receivedAt, report.server, report.receivedAt);
}
return report;
})();
+75 -18
View File
@@ -6,41 +6,98 @@ import { getLibraryFile, libraryFilesRetired } from './index.js';
import { assertRegularPath, parseByteRange } from './storage.js';
export function gameFileError(cause: unknown): Response {
const status = typeof cause === 'object' && cause !== null && 'status' in cause && typeof cause.status === 'number' ? cause.status : 400;
const receivedBytes = typeof cause === 'object' && cause !== null && 'receivedBytes' in cause && Number.isSafeInteger(cause.receivedBytes) ? cause.receivedBytes : undefined;
return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.', ...(receivedBytes === undefined ? {} : { receivedBytes }) }, { status });
const status =
typeof cause === 'object' &&
cause !== null &&
'status' in cause &&
typeof cause.status === 'number'
? cause.status
: 400;
const receivedBytes =
typeof cause === 'object' &&
cause !== null &&
'receivedBytes' in cause &&
Number.isSafeInteger(cause.receivedBytes)
? cause.receivedBytes
: undefined;
return json(
{
error: cause instanceof Error ? cause.message : 'Game file request failed.',
...(receivedBytes === undefined ? {} : { receivedBytes })
},
{ status }
);
}
export async function readGameJson(request: Request, maxBytes = 64 * 1024): Promise<Record<string, unknown>> {
export async function readGameJson(
request: Request,
maxBytes = 64 * 1024
): Promise<Record<string, unknown>> {
if (!request.body) throw new Error('A JSON request body is required.');
const reader = request.body.getReader();
const chunks: Uint8Array[] = []; let bytes = 0;
const chunks: Uint8Array[] = [];
let bytes = 0;
try {
while (true) {
const next = await reader.read(); if (next.done) break;
const next = await reader.read();
if (next.done) break;
bytes += next.value.byteLength;
if (bytes > maxBytes) { await reader.cancel(); throw new Error('Request body exceeds its size limit.'); }
if (bytes > maxBytes) {
await reader.cancel();
throw new Error('Request body exceeds its size limit.');
}
chunks.push(next.value);
}
} finally { reader.releaseLock(); }
} finally {
reader.releaseLock();
}
const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
if (!body || typeof body !== 'object' || Array.isArray(body)) throw new Error('Expected a JSON object.');
if (!body || typeof body !== 'object' || Array.isArray(body))
throw new Error('Expected a JSON object.');
return body;
}
export async function serveGameFile(request: Request, versionId: string, requestedPath: string): Promise<Response> {
if (libraryFilesRetired(versionId)) return json({ error: 'These outdated game files have been removed.' }, { status: 410 });
export async function serveGameFile(
request: Request,
versionId: string,
requestedPath: string
): Promise<Response> {
if (libraryFilesRetired(versionId))
return json({ error: 'These outdated game files have been removed.' }, { status: 410 });
const file = getLibraryFile(versionId, requestedPath);
if (!file || file.state !== 'ready') return json({ error: 'File is not in a published manifest.' }, { status: 404 });
if (!file || file.state !== 'ready')
return json({ error: 'File is not in a published manifest.' }, { status: 404 });
const path = await assertRegularPath(file.storageLocation, file.path);
if ((await stat(path)).size !== file.size) return json({ error: 'Stored file failed size verification.' }, { status: 503 });
if ((await stat(path)).size !== file.size)
return json({ error: 'Stored file failed size verification.' }, { status: 503 });
const etag = `"${file.sha256}"`;
const headers = new Headers({ 'Content-Type': 'application/octet-stream', 'Accept-Ranges': 'bytes', ETag: etag, 'Cache-Control': 'private, no-cache', 'X-Content-SHA256': file.sha256 });
if (request.headers.get('if-none-match') === etag) return new Response(null, { status: 304, headers });
const headers = new Headers({
'Content-Type': 'application/octet-stream',
'Accept-Ranges': 'bytes',
ETag: etag,
'Cache-Control': 'private, no-cache',
'X-Content-SHA256': file.sha256
});
if (request.headers.get('if-none-match') === etag)
return new Response(null, { status: 304, headers });
let range;
try { range = parseByteRange(request.headers.has('if-range') && request.headers.get('if-range') !== etag ? null : request.headers.get('range'), file.size); }
catch { headers.set('Content-Range', `bytes */${file.size}`); return new Response(null, { status: 416, headers }); }
try {
range = parseByteRange(
request.headers.has('if-range') && request.headers.get('if-range') !== etag
? null
: request.headers.get('range'),
file.size
);
} catch {
headers.set('Content-Range', `bytes */${file.size}`);
return new Response(null, { status: 416, headers });
}
headers.set('Content-Length', String(range ? range.end - range.start + 1 : file.size));
if (range) headers.set('Content-Range', `bytes ${range.start}-${range.end}/${file.size}`);
const body = request.method === 'HEAD' ? null : Readable.toWeb(createReadStream(path, range ?? undefined)) as ReadableStream<Uint8Array>;
const body =
request.method === 'HEAD'
? null
: (Readable.toWeb(
createReadStream(path, range ?? undefined)
) as ReadableStream<Uint8Array>);
return new Response(body, { status: range ? 206 : 200, headers });
}
File diff suppressed because it is too large Load Diff
+12 -5
View File
@@ -50,10 +50,17 @@ export function ensureGameFileSchema(handle: Database.Database): void {
WHEN OLD.state = 'ready' BEGIN SELECT RAISE(ABORT, 'Published game-file versions are immutable'); END;
CREATE INDEX IF NOT EXISTS idx_host_library_caches_version ON host_library_caches(version_id);
`);
const versionColumns = handle.prepare('PRAGMA table_info(game_file_versions)').all() as { name: string }[];
if (!versionColumns.some(column => column.name === 'last_activity_at')) handle.exec('ALTER TABLE game_file_versions ADD COLUMN last_activity_at TEXT');
const cacheColumns = handle.prepare('PRAGMA table_info(host_library_caches)').all() as { name: string }[];
if (!cacheColumns.some(column => column.name === 'bytes_verified')) handle.exec('ALTER TABLE host_library_caches ADD COLUMN bytes_verified INTEGER');
if (!cacheColumns.some(column => column.name === 'transfer_progress')) handle.exec('ALTER TABLE host_library_caches ADD COLUMN transfer_progress TEXT');
const versionColumns = handle.prepare('PRAGMA table_info(game_file_versions)').all() as {
name: string;
}[];
if (!versionColumns.some((column) => column.name === 'last_activity_at'))
handle.exec('ALTER TABLE game_file_versions ADD COLUMN last_activity_at TEXT');
const cacheColumns = handle.prepare('PRAGMA table_info(host_library_caches)').all() as {
name: string;
}[];
if (!cacheColumns.some((column) => column.name === 'bytes_verified'))
handle.exec('ALTER TABLE host_library_caches ADD COLUMN bytes_verified INTEGER');
if (!cacheColumns.some((column) => column.name === 'transfer_progress'))
handle.exec('ALTER TABLE host_library_caches ADD COLUMN transfer_progress TEXT');
initialized.add(handle);
}
+104 -35
View File
@@ -8,39 +8,62 @@ import { randomUUID } from 'node:crypto';
import type { GameFileEntry } from '../../game-files.js';
export function validateGamePath(input: unknown): string {
if (typeof input !== 'string' || !input || input.length > 1024 || /[\\\x00-\x1f\x7f:]/.test(input) || Buffer.from(input).toString('utf8') !== input
|| isAbsolute(input) || input.split('/').some(part => !part || part === '.' || part === '..' || /[. ]$/.test(part))) {
if (
typeof input !== 'string' ||
!input ||
input.length > 1024 ||
// eslint-disable-next-line no-control-regex -- Reject control characters in game file paths.
/[\\\x00-\x1f\x7f:]/.test(input) ||
Buffer.from(input).toString('utf8') !== input ||
isAbsolute(input) ||
input
.split('/')
.some((part) => !part || part === '.' || part === '..' || /[. ]$/.test(part))
) {
throw new Error('Game file paths must be relative, normalized paths without traversal.');
}
return input;
}
export function validateManifest(input: unknown): GameFileEntry[] {
if (!Array.isArray(input) || !input.length || input.length > 200_000) throw new Error('Provide a manifest containing 1–200000 files.');
if (!Array.isArray(input) || !input.length || input.length > 200_000)
throw new Error('Provide a manifest containing 1–200000 files.');
const paths = new Set<string>();
let total = 0;
const files = input.map(raw => {
if (!raw || typeof raw !== 'object') throw new Error('Invalid manifest entry.');
const path = validateGamePath(raw.path);
if (!Number.isSafeInteger(raw.size) || raw.size < 0 || typeof raw.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(raw.sha256)) throw new Error(`Invalid size or SHA256 for ${path}.`);
const key = path.toLowerCase();
if (paths.has(key)) throw new Error(`Duplicate game file path: ${path}.`);
paths.add(key); total += raw.size;
if (!Number.isSafeInteger(total)) throw new Error('Library size exceeds the supported limit.');
return { path, size: raw.size as number, sha256: raw.sha256 as string };
}).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
const files = input
.map((raw) => {
if (!raw || typeof raw !== 'object') throw new Error('Invalid manifest entry.');
const path = validateGamePath(raw.path);
if (
!Number.isSafeInteger(raw.size) ||
raw.size < 0 ||
typeof raw.sha256 !== 'string' ||
!/^[a-f0-9]{64}$/.test(raw.sha256)
)
throw new Error(`Invalid size or SHA256 for ${path}.`);
const key = path.toLowerCase();
if (paths.has(key)) throw new Error(`Duplicate game file path: ${path}.`);
paths.add(key);
total += raw.size;
if (!Number.isSafeInteger(total))
throw new Error('Library size exceeds the supported limit.');
return { path, size: raw.size as number, sha256: raw.sha256 as string };
})
.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
for (const file of files) {
const parts = file.path.toLowerCase().split('/');
parts.pop();
while (parts.length) {
if (paths.has(parts.join('/'))) throw new Error('A manifest path cannot be both a file and a directory.');
if (paths.has(parts.join('/')))
throw new Error('A manifest path cannot be both a file and a directory.');
parts.pop();
}
}
return files;
}
export const manifestChecksum = (files: GameFileEntry[]): string => createHash('sha256').update(JSON.stringify(files)).digest('hex');
export const manifestChecksum = (files: GameFileEntry[]): string =>
createHash('sha256').update(JSON.stringify(files)).digest('hex');
export function gameFilePath(root: string, path: string): string {
const base = resolve(root);
@@ -50,21 +73,41 @@ export function gameFilePath(root: string, path: string): string {
}
/** Refuse symlinks at every existing component, including the managed root. */
export async function assertRegularPath(root: string, path: string, allowMissing = false): Promise<string> {
export async function assertRegularPath(
root: string,
path: string,
allowMissing = false
): Promise<string> {
const result = gameFilePath(root, path);
let current = resolve(root);
const components = [current, ...validateGamePath(path).split('/').map(part => current = resolve(current, part))];
const components = [
current,
...validateGamePath(path)
.split('/')
.map((part) => (current = resolve(current, part)))
];
for (let i = 0; i < components.length; i++) {
let info;
try { info = await lstat(components[i]); }
catch (cause) { if (allowMissing && (cause as NodeJS.ErrnoException).code === 'ENOENT') continue; throw cause; }
if (info.isSymbolicLink() || (i < components.length - 1 ? !info.isDirectory() : !info.isFile())) throw new Error('Game storage contains a non-regular path.');
try {
info = await lstat(components[i]);
} catch (cause) {
if (allowMissing && (cause as NodeJS.ErrnoException).code === 'ENOENT') continue;
throw cause;
}
if (
info.isSymbolicLink() ||
(i < components.length - 1 ? !info.isDirectory() : !info.isFile())
)
throw new Error('Game storage contains a non-regular path.');
}
return result;
}
/** Compute a file digest without buffering game archives in memory. */
export async function hashGameFile(root: string, file: Pick<GameFileEntry, 'path' | 'size'>): Promise<string> {
export async function hashGameFile(
root: string,
file: Pick<GameFileEntry, 'path' | 'size'>
): Promise<string> {
const path = await assertRegularPath(root, file.path);
const info = await stat(path);
if (info.size !== file.size) throw new Error(`Size verification failed for ${file.path}.`);
@@ -74,35 +117,61 @@ export async function hashGameFile(root: string, file: Pick<GameFileEntry, 'path
}
export async function verifyGameFile(root: string, file: GameFileEntry): Promise<void> {
if (await hashGameFile(root, file) !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`);
if ((await hashGameFile(root, file)) !== file.sha256)
throw new Error(`Checksum verification failed for ${file.path}.`);
}
/** Stream to a private temporary file; publish only after byte count and SHA256 match. */
export async function receiveGameFile(root: string, file: GameFileEntry, body: ReadableStream<Uint8Array> | null): Promise<void> {
export async function receiveGameFile(
root: string,
file: GameFileEntry,
body: ReadableStream<Uint8Array> | null
): Promise<void> {
if (!body) throw new Error('A file body is required.');
const path = await assertRegularPath(root, file.path, true);
await mkdir(dirname(path), { recursive: true });
const temp = `${path}.upload-${randomUUID()}`;
let size = 0;
const hash = createHash('sha256');
const verifier = new Transform({ transform(chunk, _encoding, done) {
size += chunk.length;
if (size > file.size) return done(new Error('Upload exceeds the manifest size.'));
hash.update(chunk); done(null, chunk);
} });
const verifier = new Transform({
transform(chunk, _encoding, done) {
size += chunk.length;
if (size > file.size) return done(new Error('Upload exceeds the manifest size.'));
hash.update(chunk);
done(null, chunk);
}
});
try {
await pipeline(Readable.fromWeb(body as never), verifier, createWriteStream(temp, { flags: 'wx', mode: 0o600, flush: true }));
if (size !== file.size || hash.digest('hex') !== file.sha256) throw new Error(`Upload checksum or size mismatch for ${file.path}.`);
await pipeline(
Readable.fromWeb(body as never),
verifier,
createWriteStream(temp, { flags: 'wx', mode: 0o600, flush: true })
);
if (size !== file.size || hash.digest('hex') !== file.sha256)
throw new Error(`Upload checksum or size mismatch for ${file.path}.`);
await rename(temp, path);
} finally { await rm(temp, { force: true }); }
} finally {
await rm(temp, { force: true });
}
}
export function parseByteRange(header: string | null, size: number): { start: number; end: number } | null {
export function parseByteRange(
header: string | null,
size: number
): { start: number; end: number } | null {
if (!header) return null;
const match = /^bytes=(\d*)-(\d*)$/.exec(header);
if (!match || (!match[1] && !match[2]) || size === 0) throw new Error('Unsatisfiable range.');
let start = match[1] ? Number(match[1]) : Math.max(0, size - Number(match[2]));
let end = match[1] && match[2] ? Number(match[2]) : size - 1;
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || start >= size || end < start || (!match[1] && Number(match[2]) === 0)) throw new Error('Unsatisfiable range.');
const start = match[1] ? Number(match[1]) : Math.max(0, size - Number(match[2]));
const end = match[1] && match[2] ? Number(match[2]) : size - 1;
if (
!Number.isSafeInteger(start) ||
!Number.isSafeInteger(end) ||
start < 0 ||
start >= size ||
end < start ||
(!match[1] && Number(match[2]) === 0)
)
throw new Error('Unsatisfiable range.');
return { start, end: Math.min(end, size - 1) };
}
+122 -47
View File
@@ -32,28 +32,47 @@ export class UploadOffsetError extends Error {
/** Apply the same path, case, size and file/directory collision rules as published manifests. */
export function validateUploadPlan(input: unknown): UploadFilePlan[] {
if (!Array.isArray(input) || !input.length || input.length > 200_000) throw new Error('Provide a folder containing 1–200000 files.');
if (!Array.isArray(input) || !input.length || input.length > 200_000)
throw new Error('Provide a folder containing 1–200000 files.');
const modified = new Map<string, number>();
const manifest = validateManifest(input.map(raw => {
if (!raw || typeof raw !== 'object') throw new Error('Invalid upload file.');
const lastModified = raw.lastModified === undefined ? 0 : raw.lastModified;
if (!Number.isSafeInteger(lastModified) || lastModified < 0) throw new Error('Invalid file modification time.');
modified.set(raw.path, lastModified);
return { path: raw.path, size: raw.size, sha256: '0'.repeat(64) };
const manifest = validateManifest(
input.map((raw) => {
if (!raw || typeof raw !== 'object') throw new Error('Invalid upload file.');
const lastModified = raw.lastModified === undefined ? 0 : raw.lastModified;
if (!Number.isSafeInteger(lastModified) || lastModified < 0)
throw new Error('Invalid file modification time.');
modified.set(raw.path, lastModified);
return { path: raw.path, size: raw.size, sha256: '0'.repeat(64) };
})
);
return manifest.map((file) => ({
path: file.path,
size: file.size,
lastModified: modified.get(file.path)!
}));
return manifest.map(file => ({ path: file.path, size: file.size, lastModified: modified.get(file.path)! }));
}
interface UploadLocks { barrier: Promise<void>; writers: number; files: Map<string, Promise<void>>; active: Set<Promise<void>> }
interface UploadLocks {
barrier: Promise<void>;
writers: number;
files: Map<string, Promise<void>>;
active: Set<Promise<void>>;
}
const uploadLocks = new Map<string, UploadLocks>();
export function uploadIsActive(root: string): boolean { return uploadLocks.has(resolve(root)); }
export function uploadIsActive(root: string): boolean {
return uploadLocks.has(resolve(root));
}
function locksFor(key: string): UploadLocks {
let locks = uploadLocks.get(key);
if (!locks) { locks = { barrier: Promise.resolve(), writers: 0, files: new Map(), active: new Set() }; uploadLocks.set(key, locks); }
if (!locks) {
locks = { barrier: Promise.resolve(), writers: 0, files: new Map(), active: new Set() };
uploadLocks.set(key, locks);
}
return locks;
}
function releaseLocks(key: string, locks: UploadLocks) {
if (!locks.writers && !locks.active.size && uploadLocks.get(key) === locks) uploadLocks.delete(key);
if (!locks.writers && !locks.active.size && uploadLocks.get(key) === locks)
uploadLocks.delete(key);
}
/** Serialize transfers and final publication for one import; a rejected operation never poisons the lock. */
@@ -64,12 +83,15 @@ export async function withUploadLock<T>(root: string, callback: () => Promise<T>
// from starting until publication finishes (including failed publication).
const previous = Promise.all([locks.barrier, ...locks.active]);
let release!: () => void;
const current = new Promise<void>(done => { release = done; });
const current = new Promise<void>((done) => {
release = done;
});
locks.barrier = current;
locks.writers++;
await previous;
try { return await callback(); }
finally {
try {
return await callback();
} finally {
release();
locks.writers--;
releaseLocks(key, locks);
@@ -77,31 +99,47 @@ export async function withUploadLock<T>(root: string, callback: () => Promise<T>
}
/** Different files may transfer together; offsets for the same file remain serial. */
export async function withUploadFileLock<T>(root: string, file: string, callback: () => Promise<T>): Promise<T> {
const key = resolve(root), locks = locksFor(key);
export async function withUploadFileLock<T>(
root: string,
file: string,
callback: () => Promise<T>
): Promise<T> {
const key = resolve(root),
locks = locksFor(key);
const previous = Promise.all([locks.barrier, locks.files.get(file)]);
let release!: () => void;
const current = new Promise<void>(done => { release = done; });
locks.files.set(file, current); locks.active.add(current);
const current = new Promise<void>((done) => {
release = done;
});
locks.files.set(file, current);
locks.active.add(current);
await previous;
try { return await callback(); }
finally {
release(); locks.active.delete(current);
try {
return await callback();
} finally {
release();
locks.active.delete(current);
if (locks.files.get(file) === current) locks.files.delete(file);
releaseLocks(key, locks);
}
}
/** File size is the resume checkpoint, including after a controller restart during an append. */
export async function getUploadFileProgress(root: string, file: Pick<UploadFilePlan, 'path' | 'size'>): Promise<UploadFileProgress> {
export async function getUploadFileProgress(
root: string,
file: Pick<UploadFilePlan, 'path' | 'size'>
): Promise<UploadFileProgress> {
const path = await assertRegularPath(root, file.path, true);
try {
const info = await lstat(path);
if (!info.isFile() || info.isSymbolicLink()) throw new Error('Game storage contains a non-regular path.');
if (info.size > file.size) throw new Error(`Stored upload exceeds the declared size of ${file.path}.`);
if (!info.isFile() || info.isSymbolicLink())
throw new Error('Game storage contains a non-regular path.');
if (info.size > file.size)
throw new Error(`Stored upload exceeds the declared size of ${file.path}.`);
return { receivedBytes: info.size, complete: info.size === file.size };
} catch (cause) {
if ((cause as NodeJS.ErrnoException).code === 'ENOENT') return { receivedBytes: 0, complete: false };
if ((cause as NodeJS.ErrnoException).code === 'ENOENT')
return { receivedBytes: 0, complete: false };
throw cause;
}
}
@@ -118,10 +156,17 @@ export async function receiveUploadChunk(
body: ReadableStream<Uint8Array> | null,
beforeWrite?: () => Promise<void> | void
): Promise<UploadFileProgress> {
if (!Number.isSafeInteger(chunk.offset) || chunk.offset < 0 || !Number.isSafeInteger(chunk.size)
|| chunk.size < 0 || chunk.size > MAX_UPLOAD_CHUNK_BYTES || chunk.offset + chunk.size > file.size
|| !Number.isSafeInteger(chunk.offset + chunk.size) || !/^[a-f0-9]{64}$/.test(chunk.sha256)
|| (chunk.size === 0 && file.size !== 0)) {
if (
!Number.isSafeInteger(chunk.offset) ||
chunk.offset < 0 ||
!Number.isSafeInteger(chunk.size) ||
chunk.size < 0 ||
chunk.size > MAX_UPLOAD_CHUNK_BYTES ||
chunk.offset + chunk.size > file.size ||
!Number.isSafeInteger(chunk.offset + chunk.size) ||
!/^[a-f0-9]{64}$/.test(chunk.sha256) ||
(chunk.size === 0 && file.size !== 0)
) {
throw new Error('Invalid upload chunk size, offset or SHA256.');
}
if (!body && chunk.size !== 0) throw new Error('A file chunk body is required.');
@@ -129,7 +174,8 @@ export async function receiveUploadChunk(
return withUploadFileLock(root, file.path, async () => {
await beforeWrite?.();
const progress = await getUploadFileProgress(root, file);
if (chunk.offset !== progress.receivedBytes) throw new UploadOffsetError(progress.receivedBytes);
if (chunk.offset !== progress.receivedBytes)
throw new UploadOffsetError(progress.receivedBytes);
const destination = await assertRegularPath(root, file.path, true);
// Keep transient chunks outside the tree that will become the immutable library.
const chunkRoot = `${resolve(root)}.chunks`;
@@ -137,19 +183,31 @@ export async function receiveUploadChunk(
await mkdir(dirname(temporary), { recursive: true });
let received = 0;
const hash = createHash('sha256');
const verifier = new Transform({ transform(data, _encoding, done) {
received += data.length;
if (received > chunk.size) return done(new Error('Upload exceeds the declared chunk size.'));
hash.update(data); done(null, data);
} });
const verifier = new Transform({
transform(data, _encoding, done) {
received += data.length;
if (received > chunk.size)
return done(new Error('Upload exceeds the declared chunk size.'));
hash.update(data);
done(null, data);
}
});
try {
await pipeline(body ? Readable.fromWeb(body as never) : Readable.from([]), verifier,
createWriteStream(temporary, { flags: 'wx', mode: 0o600, flush: true }));
if (received !== chunk.size || hash.digest('hex') !== chunk.sha256) throw new Error('Upload chunk checksum or size mismatch.');
await pipeline(
body ? Readable.fromWeb(body as never) : Readable.from([]),
verifier,
createWriteStream(temporary, { flags: 'wx', mode: 0o600, flush: true })
);
if (received !== chunk.size || hash.digest('hex') !== chunk.sha256)
throw new Error('Upload chunk checksum or size mismatch.');
await mkdir(dirname(destination), { recursive: true });
await assertRegularPath(root, file.path, true);
const target = await open(destination, constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
const target = await open(
destination,
constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW,
0o600
);
try {
const info = await target.stat();
if (!info.isFile()) throw new Error('Game storage contains a non-regular path.');
@@ -160,8 +218,14 @@ export async function receiveUploadChunk(
const bytes = data as Buffer;
let written = 0;
while (written < bytes.length) {
const result = await target.write(bytes, written, bytes.length - written, position);
if (result.bytesWritten === 0) throw new Error('Unable to append the upload chunk.');
const result = await target.write(
bytes,
written,
bytes.length - written,
position
);
if (result.bytesWritten === 0)
throw new Error('Unable to append the upload chunk.');
written += result.bytesWritten;
position += result.bytesWritten;
}
@@ -172,15 +236,26 @@ export async function receiveUploadChunk(
await target.sync();
throw cause;
}
} finally { await target.close(); }
return { receivedBytes: chunk.offset + chunk.size, complete: chunk.offset + chunk.size === file.size };
} finally { await rm(temporary, { force: true }); }
} finally {
await target.close();
}
return {
receivedBytes: chunk.offset + chunk.size,
complete: chunk.offset + chunk.size === file.size
};
} finally {
await rm(temporary, { force: true });
}
});
}
/** Caller holds withUploadLock through hashing and publication to prevent concurrent writes. */
export async function hashUploadedFiles(root: string, files: ReadonlyArray<Pick<UploadFilePlan, 'path' | 'size'>>): Promise<GameFileEntry[]> {
export async function hashUploadedFiles(
root: string,
files: ReadonlyArray<Pick<UploadFilePlan, 'path' | 'size'>>
): Promise<GameFileEntry[]> {
const manifest: GameFileEntry[] = [];
for (const file of files) manifest.push({ path: file.path, size: file.size, sha256: await hashGameFile(root, file) });
for (const file of files)
manifest.push({ path: file.path, size: file.size, sha256: await hashGameFile(root, file) });
return validateManifest(manifest);
}
+169 -52
View File
@@ -29,41 +29,81 @@ const scheduled = new WeakSet<ReturnType<typeof db>>();
function snapshotSchemaReady(handle: ReturnType<typeof db>): boolean {
// Host enrollment and game-file maintenance can run before provisioning is
// initialized. Missing queue data means unknown capacity, not an empty fleet.
const { count } = handle.prepare(`SELECT COUNT(*) AS count FROM sqlite_master WHERE type='table'
AND name IN ('native_instances','native_provisioning','server_deployments','settings')`).get() as { count: number };
const { count } = handle
.prepare(
`SELECT COUNT(*) AS count FROM sqlite_master WHERE type='table'
AND name IN ('native_instances','native_provisioning','server_deployments','settings')`
)
.get() as { count: number };
if (count === 4) return true;
snapshots.delete(handle);
return false;
}
function queueSnapshot(handle: ReturnType<typeof db>, hosts: Map<string, HostAvailability>): Snapshot {
const counts = handle.prepare(`SELECT
function queueSnapshot(
handle: ReturnType<typeof db>,
hosts: Map<string, HostAvailability>
): Snapshot {
const counts = handle
.prepare(
`SELECT
COALESCE(SUM(state IN ('running','awaiting-health') OR ${committedRebalanceStartup}),0) AS active,
COALESCE(SUM(state='queued'),0) AS queued FROM server_deployments
WHERE action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup})
AND server_deleted_at IS NULL`).get() as { active: number; queued: number };
return { hosts, ...counts, startupLimit: fleetSettings().createConcurrency, updatedAt: new Date().toISOString() };
AND server_deleted_at IS NULL`
)
.get() as { active: number; queued: number };
return {
hosts,
...counts,
startupLimit: fleetSettings().createConcurrency,
updatedAt: new Date().toISOString()
};
}
function observeHost(host: Host): HostAvailability {
const handle = db();
const { running, startup } = capacityCalibration(host.id);
const measured = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample.generation]));
const pending = handle.prepare(`SELECT DISTINCT i.id,i.generation FROM native_instances i
const measured = new Map(
host.inventory.instanceResources?.map((sample) => [sample.instanceId, sample.generation])
);
const pending = handle
.prepare(
`SELECT DISTINCT i.id,i.generation FROM native_instances i
JOIN native_provisioning n ON n.instance_id=i.id JOIN server_deployments d ON d.id=n.deployment_id
WHERE i.host_id=? AND i.retired_at IS NULL AND d.action IN ('create','rebuild')
AND d.state IN ('running','awaiting-health')`).all(host.id) as { id: string; generation: number }[];
const incoming = handle.prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.target_host_id=? AND (m.removed_at IS NULL OR d.state IN ('queued','waiting','interrupted'))`).get(host.id) as { n: number };
const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length + incoming.n;
AND d.state IN ('running','awaiting-health')`
)
.all(host.id) as { id: string; generation: number }[];
const incoming = handle
.prepare(
`SELECT COUNT(*) AS n FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.target_host_id=? AND (m.removed_at IS NULL OR d.state IN ('queued','waiting','interrupted'))`
)
.get(host.id) as { n: number };
const unmeasured =
pending.filter((instance) => measured.get(instance.id) !== instance.generation).length +
incoming.n;
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
const freeCpu = cpuCount && cpuUsagePercent !== undefined
? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores) : null;
const freeMemory = memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal
? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes) : null;
return { host, freeCpu, freeMemory, runningCpu: running.cpuCores, runningMemory: running.memoryBytes,
const freeCpu =
cpuCount && cpuUsagePercent !== undefined
? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores)
: null;
const freeMemory =
memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal
? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes)
: null;
return {
host,
freeCpu,
freeMemory,
runningCpu: running.cpuCores,
runningMemory: running.memoryBytes,
calibration: { running, startup },
ports: handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as HostAvailability['ports'] };
ports: handle
.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?')
.all(host.id) as HostAvailability['ports']
};
}
/** Host telemetry updates the estimate once, independently of how many lobbies are previewed. */
@@ -85,20 +125,33 @@ export function scheduleAvailabilityRefresh(): void {
if (!handle.open) return;
try {
if (!snapshotSchemaReady(handle)) return;
const hosts = new Map(listHosts().map(host => [host.id, observeHost(host)]));
const hosts = new Map(listHosts().map((host) => [host.id, observeHost(host)]));
snapshots.set(handle, queueSnapshot(handle, hosts));
} catch (error) { console.error('[availability] snapshot refresh failed:', error); }
} catch (error) {
console.error('[availability] snapshot refresh failed:', error);
}
});
}
subscribe(event => {
if (event.type === 'fleet-invalidated' || event.type === 'server-status') scheduleAvailabilityRefresh();
subscribe((event) => {
if (event.type === 'fleet-invalidated' || event.type === 'server-status')
scheduleAvailabilityRefresh();
});
export function freePorts(snapshot: HostAvailability, bindAddress?: string): number {
const bind = bindAddress ?? snapshot.host.bindAddress;
const used = new Set(snapshot.ports.filter(row => row.port >= snapshot.host.portMin && row.port <= snapshot.host.portMax
&& (row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(row.bind_address))).map(row => row.port));
const used = new Set(
snapshot.ports
.filter(
(row) =>
row.port >= snapshot.host.portMin &&
row.port <= snapshot.host.portMax &&
(row.bind_address === bind ||
['0.0.0.0', '::'].includes(bind) ||
['0.0.0.0', '::'].includes(row.bind_address))
)
.map((row) => row.port)
);
return Math.max(0, snapshot.host.portMax - snapshot.host.portMin + 1 - used.size);
}
@@ -110,44 +163,108 @@ export function readAvailabilitySnapshot(): Snapshot | null {
}
/** Estimates only: no placement, historical scans, reservations, or live host requests. */
export function launchAvailability(selections: NativePlacement[]): { availability: LaunchAvailability; warnings: string[] } {
export function launchAvailability(selections: NativePlacement[]): {
availability: LaunchAvailability;
warnings: string[];
} {
const snapshot = snapshots.get(db());
if (!snapshot) {
scheduleAvailabilityRefresh();
return { availability: { updatedAt: null, requested: selections.length, estimatedAvailable: null, portsAvailable: null,
cpuCoresAvailable: null, memoryBytesAvailable: null, startupSlots: null, startupLimit: null, queued: null },
warnings: selections.length ? ['Capacity measurements are updating. You can queue this launch; provisioning waits for an eligible host.'] : [] };
return {
availability: {
updatedAt: null,
requested: selections.length,
estimatedAvailable: null,
portsAvailable: null,
cpuCoresAvailable: null,
memoryBytesAvailable: null,
startupSlots: null,
startupLimit: null,
queued: null
},
warnings: selections.length
? [
'Capacity measurements are updating. You can queue this launch; provisioning waits for an eligible host.'
]
: []
};
}
const candidates = [...snapshot.hosts.values()].filter(({ host }) => host.readiness === 'ready'
&& host.lastHeartbeat && Date.now() - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS
&& (host.inventory.storageAvailable ?? 0) >= host.minimumFreeBytes + 1024 ** 3);
const eligible = (candidate: HostAvailability, selection: NativePlacement) => (!selection.hostId || selection.automatic || candidate.host.id === selection.hostId)
&& !!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '')
&& !!candidate.host.inventory.libraries?.some(library => library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state));
const relevant = candidates.filter(candidate => selections.some(selection => eligible(candidate, selection)));
const candidates = [...snapshot.hosts.values()].filter(
({ host }) =>
host.readiness === 'ready' &&
host.lastHeartbeat &&
Date.now() - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS &&
(host.inventory.storageAvailable ?? 0) >= host.minimumFreeBytes + 1024 ** 3
);
const eligible = (candidate: HostAvailability, selection: NativePlacement) =>
(!selection.hostId || selection.automatic || candidate.host.id === selection.hostId) &&
!!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '') &&
!!candidate.host.inventory.libraries?.some(
(library) =>
library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state)
);
const relevant = candidates.filter((candidate) =>
selections.some((selection) => eligible(candidate, selection))
);
const planned = new Map<string, number>();
let estimatedAvailable = 0;
for (const selection of selections) {
const candidate = relevant.find(candidate => {
const candidate = relevant.find((candidate) => {
const used = planned.get(candidate.host.id) ?? 0;
return eligible(candidate, selection) && freePorts(candidate, selection.network?.bindAddress ?? selection.bindAddress) > used
&& candidate.freeCpu !== null && candidate.freeMemory !== null
&& candidate.freeCpu >= (used + 1) * candidate.runningCpu
&& candidate.freeMemory >= (used + 1) * candidate.runningMemory;
return (
eligible(candidate, selection) &&
freePorts(candidate, selection.network?.bindAddress ?? selection.bindAddress) >
used &&
candidate.freeCpu !== null &&
candidate.freeMemory !== null &&
candidate.freeCpu >= (used + 1) * candidate.runningCpu &&
candidate.freeMemory >= (used + 1) * candidate.runningMemory
);
});
if (candidate) { estimatedAvailable++; planned.set(candidate.host.id, (planned.get(candidate.host.id) ?? 0) + 1); }
if (candidate) {
estimatedAvailable++;
planned.set(candidate.host.id, (planned.get(candidate.host.id) ?? 0) + 1);
}
}
const warnings: string[] = [];
const unknown = relevant.some(candidate => candidate.freeCpu === null || candidate.freeMemory === null);
if (selections.length && !relevant.length) warnings.push('Waiting for an eligible host with the selected runtime and game files. The launch can be queued now.');
else if (unknown) warnings.push('Some host CPU or memory measurements are unavailable. Provisioning waits for fresh capacity measurements.');
else if (estimatedAvailable < selections.length) warnings.push(`Current capacity is estimated for ${estimatedAvailable} of ${selections.length} servers. Remaining launches wait for capacity, including any removals in this collection.`);
const unknown = relevant.some(
(candidate) => candidate.freeCpu === null || candidate.freeMemory === null
);
if (selections.length && !relevant.length)
warnings.push(
'Waiting for an eligible host with the selected runtime and game files. The launch can be queued now.'
);
else if (unknown)
warnings.push(
'Some host CPU or memory measurements are unavailable. Provisioning waits for fresh capacity measurements.'
);
else if (estimatedAvailable < selections.length)
warnings.push(
`Current capacity is estimated for ${estimatedAvailable} of ${selections.length} servers. Remaining launches wait for capacity, including any removals in this collection.`
);
const startupSlots = Math.max(0, snapshot.startupLimit - snapshot.active);
if (selections.length && !startupSlots) warnings.push('All startup slots are in use. New launches will wait in the queue.');
if (selections.length && snapshot.queued) warnings.push(`${snapshot.queued} server${snapshot.queued === 1 ? ' is' : 's are'} already queued.`);
return { availability: { updatedAt: snapshot.updatedAt, requested: selections.length, estimatedAvailable: unknown ? null : estimatedAvailable,
portsAvailable: relevant.reduce((total, candidate) => total + freePorts(candidate), 0),
cpuCoresAvailable: unknown ? null : relevant.reduce((total, candidate) => total + (candidate.freeCpu ?? 0), 0),
memoryBytesAvailable: unknown ? null : relevant.reduce((total, candidate) => total + (candidate.freeMemory ?? 0), 0),
startupSlots, startupLimit: snapshot.startupLimit, queued: snapshot.queued }, warnings };
if (selections.length && !startupSlots)
warnings.push('All startup slots are in use. New launches will wait in the queue.');
if (selections.length && snapshot.queued)
warnings.push(
`${snapshot.queued} server${snapshot.queued === 1 ? ' is' : 's are'} already queued.`
);
return {
availability: {
updatedAt: snapshot.updatedAt,
requested: selections.length,
estimatedAvailable: unknown ? null : estimatedAvailable,
portsAvailable: relevant.reduce((total, candidate) => total + freePorts(candidate), 0),
cpuCoresAvailable: unknown
? null
: relevant.reduce((total, candidate) => total + (candidate.freeCpu ?? 0), 0),
memoryBytesAvailable: unknown
? null
: relevant.reduce((total, candidate) => total + (candidate.freeMemory ?? 0), 0),
startupSlots,
startupLimit: snapshot.startupLimit,
queued: snapshot.queued
},
warnings
};
}
+185 -57
View File
@@ -1,7 +1,13 @@
import { fleetSettings } from '../settings.js';
import { db } from '../db/index.js';
import { ensureHostSchema } from './schema.js';
import type { CapacityCalibration, CapacitySample, CapacitySamplePage, HostInventory, ResourceAllowance } from '../../hosts.js';
import type {
CapacityCalibration,
CapacitySample,
CapacitySamplePage,
HostInventory,
ResourceAllowance
} from '../../hosts.js';
const BOOTSTRAP_CPU = 0.5;
const BOOTSTRAP_MEMORY = 1024 ** 3;
@@ -9,59 +15,114 @@ const WINDOW_MS = 24 * 60 * 60_000;
const MAX_SAMPLES = 10_000;
type Phase = 'startup' | 'running';
interface AllowanceRow {
scope: string; phase: Phase; cpu_cores: number; memory_bytes: number; samples: number;
window_start: string; observed_at: string; computed_at: string;
scope: string;
phase: Phase;
cpu_cores: number;
memory_bytes: number;
samples: number;
window_start: string;
observed_at: string;
computed_at: string;
}
function database() {
const handle = db();
ensureHostSchema(handle);
return handle;
}
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
/** Read retained minute peaks without contacting agents or recalculating allowances. */
export function capacitySamples(hostId: string, options: { scope: 'host' | 'fleet'; phase: 'all' | Phase; page: number }): CapacitySamplePage {
export function capacitySamples(
hostId: string,
options: { scope: 'host' | 'fleet'; phase: 'all' | Phase; page: number }
): CapacitySamplePage {
const handle = database();
const clauses = ['s.bucket >= ?'];
const values: (string | number)[] = [new Date(Date.now() - WINDOW_MS).toISOString()];
if (options.scope === 'host') { clauses.push('s.host_id = ?'); values.push(hostId); }
if (options.phase !== 'all') { clauses.push('s.phase = ?'); values.push(options.phase); }
if (options.scope === 'host') {
clauses.push('s.host_id = ?');
values.push(hostId);
}
if (options.phase !== 'all') {
clauses.push('s.phase = ?');
values.push(options.phase);
}
const where = clauses.join(' AND ');
const total = (handle.prepare(`SELECT COUNT(*) AS count FROM ec_instance_resource_samples s WHERE ${where}`).get(...values) as { count: number }).count;
const total = (
handle
.prepare(`SELECT COUNT(*) AS count FROM ec_instance_resource_samples s WHERE ${where}`)
.get(...values) as { count: number }
).count;
const pageSize = 50;
const page = Math.min(options.page, Math.max(1, Math.ceil(total / pageSize)));
const samples = handle.prepare(`SELECT s.host_id AS hostId, h.name AS hostName, s.instance_id AS instanceId,
const samples = handle
.prepare(
`SELECT s.host_id AS hostId, h.name AS hostName, s.instance_id AS instanceId,
i.server, s.generation, s.phase, s.bucket AS timestamp, s.cpu_cores AS cpuCores, s.memory_bytes AS memoryBytes
FROM ec_instance_resource_samples s
LEFT JOIN ec_hosts h ON h.id=s.host_id
LEFT JOIN native_instances i ON i.id=s.instance_id AND i.host_id=s.host_id
WHERE ${where} ORDER BY s.bucket DESC, s.host_id, s.instance_id, s.generation, s.phase LIMIT ? OFFSET ?`)
WHERE ${where} ORDER BY s.bucket DESC, s.host_id, s.instance_id, s.generation, s.phase LIMIT ? OFFSET ?`
)
.all(...values, pageSize, (page - 1) * pageSize) as CapacitySample[];
return { samples, total, page, pageSize };
}
export function validateInstanceResources(inventory: HostInventory): void {
if (inventory.instanceResourcesError !== undefined && (typeof inventory.instanceResourcesError !== 'string'
|| inventory.instanceResourcesError.length > 500)) throw new Error('Invalid instance resource sampling error.');
if (
inventory.instanceResourcesError !== undefined &&
(typeof inventory.instanceResourcesError !== 'string' ||
inventory.instanceResourcesError.length > 500)
)
throw new Error('Invalid instance resource sampling error.');
const samples = inventory.instanceResources;
if (samples === undefined) return;
if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.');
if (!Array.isArray(samples) || samples.length > 1000)
throw new Error('Invalid instance resource samples.');
const seen = new Set<string>();
for (const sample of samples) {
if (!sample || typeof sample !== 'object' || typeof sample.instanceId !== 'string' || !sample.instanceId || sample.instanceId.length > 100
|| !Number.isSafeInteger(sample.generation) || sample.generation < 1
|| !Number.isSafeInteger(sample.uptimeSeconds) || sample.uptimeSeconds < 0
|| typeof sample.cpuCores !== 'number' || !Number.isFinite(sample.cpuCores) || sample.cpuCores < 0
|| !Number.isSafeInteger(sample.memoryBytes) || sample.memoryBytes <= 0
|| (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01)
|| (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal)
|| seen.has(sample.instanceId)) throw new Error('Invalid instance resource sample.');
if (
!sample ||
typeof sample !== 'object' ||
typeof sample.instanceId !== 'string' ||
!sample.instanceId ||
sample.instanceId.length > 100 ||
!Number.isSafeInteger(sample.generation) ||
sample.generation < 1 ||
!Number.isSafeInteger(sample.uptimeSeconds) ||
sample.uptimeSeconds < 0 ||
typeof sample.cpuCores !== 'number' ||
!Number.isFinite(sample.cpuCores) ||
sample.cpuCores < 0 ||
!Number.isSafeInteger(sample.memoryBytes) ||
sample.memoryBytes <= 0 ||
(inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01) ||
(inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal) ||
seen.has(sample.instanceId)
)
throw new Error('Invalid instance resource sample.');
seen.add(sample.instanceId);
}
}
/** Heartbeat ingestion owns all learning work. Launch requests only read cached allowances. */
export function recordInstanceResources(hostId: string, inventory: HostInventory, timestamp: string): void {
if (!inventory.instanceResources?.length || !inventory.dockerRunning || !inventory.cpuCount || !inventory.memoryTotal) return;
export function recordInstanceResources(
hostId: string,
inventory: HostInventory,
timestamp: string
): void {
if (
!inventory.instanceResources?.length ||
!inventory.dockerRunning ||
!inventory.cpuCount ||
!inventory.memoryTotal
)
return;
const handle = database();
const bucket = timestamp.slice(0, 16) + ':00.000Z';
const observed = new Map<Phase, { cpu: number; memory: number }>();
const owned = handle.prepare('SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL');
const owned = handle.prepare(
'SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL'
);
const insert = handle.prepare(`INSERT INTO ec_instance_resource_samples
(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,?,?,?,?,?)
ON CONFLICT(host_id,instance_id,generation,phase,bucket) DO UPDATE SET
@@ -71,58 +132,125 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory
// retired and delayed pre-rebuild samples cannot contaminate the model.
if (!owned.get(sample.instanceId, hostId, sample.generation)) continue;
const phase: Phase = sample.uptimeSeconds < 300 ? 'startup' : 'running';
insert.run(hostId, sample.instanceId, sample.generation, phase, bucket, sample.cpuCores, sample.memoryBytes);
insert.run(
hostId,
sample.instanceId,
sample.generation,
phase,
bucket,
sample.cpuCores,
sample.memoryBytes
);
const prior = observed.get(phase);
observed.set(phase, { cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) });
observed.set(phase, {
cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores),
memory: Math.max(prior?.memory ?? 0, sample.memoryBytes)
});
}
for (const [phase, peak] of observed) for (const scope of [hostId, '*']) {
const previous = handle.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?').get(scope, phase) as AllowanceRow | undefined;
if (!previous || previous.computed_at < bucket) {
const rows = handle.prepare(`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples
WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
.all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
{ cpu_cores: number; memory_bytes: number; bucket: string }[];
const percentile = (values: number[]) => values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1];
const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu);
const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory));
handle.prepare(`INSERT INTO ec_resource_allowances
for (const [phase, peak] of observed)
for (const scope of [hostId, '*']) {
const previous = handle
.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?')
.get(scope, phase) as AllowanceRow | undefined;
if (!previous || previous.computed_at < bucket) {
const rows = handle
.prepare(
`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples
WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`
)
.all(
...(scope === '*' ? [] : [scope]),
phase,
new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(),
MAX_SAMPLES
) as { cpu_cores: number; memory_bytes: number; bucket: string }[];
const percentile = (values: number[]) =>
values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1];
const cpu = Math.max(percentile(rows.map((row) => row.cpu_cores)), peak.cpu);
const memory = Math.ceil(
Math.max(percentile(rows.map((row) => row.memory_bytes)), peak.memory)
);
handle
.prepare(
`INSERT INTO ec_resource_allowances
(scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET
cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples,
window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`)
.run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp);
if (scope === '*' && fleetSettings().instanceResourceKeepDays) handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket<?')
.run(new Date(Date.parse(timestamp) - Math.max(WINDOW_MS, fleetSettings().instanceResourceKeepDays * 86400_000)).toISOString());
} else {
// Raise the allowance immediately on a new peak; a lower sample
// cannot reduce it until the next scheduled percentile calculation.
handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`)
.run(peak.cpu, Math.ceil(peak.memory), timestamp, scope, phase);
window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`
)
.run(
scope,
phase,
cpu,
memory,
rows.length,
rows.at(-1)!.bucket,
timestamp,
timestamp
);
if (scope === '*' && fleetSettings().instanceResourceKeepDays)
handle
.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket<?')
.run(
new Date(
Date.parse(timestamp) -
Math.max(
WINDOW_MS,
fleetSettings().instanceResourceKeepDays * 86400_000
)
).toISOString()
);
} else {
// Raise the allowance immediately on a new peak; a lower sample
// cannot reduce it until the next scheduled percentile calculation.
handle
.prepare(
`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`
)
.run(peak.cpu, Math.ceil(peak.memory), timestamp, scope, phase);
}
}
}
}
/** Host measurements account for hardware differences; fleet evidence covers new hosts. */
export function capacityCalibration(hostId: string): CapacityCalibration {
const rows = database().prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?')
const rows = database()
.prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?')
.all(hostId, '*', new Date(Date.now() - WINDOW_MS).toISOString()) as AllowanceRow[];
const allowance = (phase: Phase): ResourceAllowance => {
const local = rows.find(row => row.scope === hostId && row.phase === phase);
const fleet = rows.find(row => row.scope === '*' && row.phase === phase);
const learned = (row?: AllowanceRow) => !!row && row.samples >= (phase === 'startup' ? 3 : 20)
&& Date.parse(row.observed_at) - Date.parse(row.window_start) >= (phase === 'startup' ? 120_000 : 300_000);
const local = rows.find((row) => row.scope === hostId && row.phase === phase);
const fleet = rows.find((row) => row.scope === '*' && row.phase === phase);
const learned = (row?: AllowanceRow) =>
!!row &&
row.samples >= (phase === 'startup' ? 3 : 20) &&
Date.parse(row.observed_at) - Date.parse(row.window_start) >=
(phase === 'startup' ? 120_000 : 300_000);
const selected = learned(local) ? local : learned(fleet) ? fleet : undefined;
return {
// A few higher observations can increase a fleet/bootstrap allowance
// immediately. Decreasing it requires the evidence thresholds above.
cpuCores: Math.max(0.01, selected?.cpu_cores ?? BOOTSTRAP_CPU, local?.cpu_cores ?? 0, selected ? 0 : fleet?.cpu_cores ?? 0),
memoryBytes: Math.max(selected?.memory_bytes ?? BOOTSTRAP_MEMORY, local?.memory_bytes ?? 0, selected ? 0 : fleet?.memory_bytes ?? 0),
source: selected ? selected.scope === hostId ? 'host' : 'fleet' : 'bootstrap',
cpuCores: Math.max(
0.01,
selected?.cpu_cores ?? BOOTSTRAP_CPU,
local?.cpu_cores ?? 0,
selected ? 0 : (fleet?.cpu_cores ?? 0)
),
memoryBytes: Math.max(
selected?.memory_bytes ?? BOOTSTRAP_MEMORY,
local?.memory_bytes ?? 0,
selected ? 0 : (fleet?.memory_bytes ?? 0)
),
source: selected ? (selected.scope === hostId ? 'host' : 'fleet') : 'bootstrap',
samples: selected?.samples ?? local?.samples ?? fleet?.samples ?? 0,
observedAt: selected?.observed_at ?? local?.observed_at ?? fleet?.observed_at ?? null
};
};
const running = allowance('running'); const startup = allowance('startup');
if (running.source === 'bootstrap' && (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)) startup.source = 'bootstrap';
const running = allowance('running');
const startup = allowance('startup');
if (
running.source === 'bootstrap' &&
(running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)
)
startup.source = 'bootstrap';
startup.cpuCores = Math.max(startup.cpuCores, running.cpuCores);
startup.memoryBytes = Math.max(startup.memoryBytes, running.memoryBytes);
return { running, startup };
+13 -4
View File
@@ -15,8 +15,11 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
const { duration, bucketMs } = windows[query.range];
const startAt = now - duration;
const count = duration / bucketMs;
const hosts = handle.prepare(`SELECT id,created_at FROM ec_hosts
WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`)
const hosts = handle
.prepare(
`SELECT id,created_at FROM ec_hosts
WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`
)
.all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[];
// The primary key bounds each host scan to the selected interval. Aggregate
// before crossing the worker boundary; do not return raw heartbeat rows.
@@ -42,8 +45,14 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
for (const point of points) {
if (createdAt <= Math.min(now, point.at + bucketMs)) point.expectedHosts++;
}
const buckets = samples.all(count - 1, startAt, bucketMs, host.id,
new Date(startAt).toISOString(), new Date(now).toISOString()) as { bucket: number; cpu: number; memory: number }[];
const buckets = samples.all(
count - 1,
startAt,
bucketMs,
host.id,
new Date(startAt).toISOString(),
new Date(now).toISOString()
) as { bucket: number; cpu: number; memory: number }[];
for (const bucket of buckets) {
const point = points[bucket.bucket];
if (!point || createdAt > Math.min(now, point.at + bucketMs)) continue;
+153 -48
View File
@@ -14,56 +14,161 @@ export async function readCapacityCurrent(query: CapacityQuery): Promise<Capacit
await yieldToController();
snapshot = readAvailabilitySnapshot();
}
if (!snapshot) return { updatedAt: null, hosts: [], hostOptions: [], warnings: ['Capacity measurements are updating.'],
summary: { estimatedServers: null, cpuCoresAvailable: null, memoryBytesAvailable: null, portsAvailable: null,
eligibleHosts: 0, totalHosts: 0, startupSlots: null, startupLimit: null, activeStartups: null, queued: null } };
if (!snapshot)
return {
updatedAt: null,
hosts: [],
hostOptions: [],
warnings: ['Capacity measurements are updating.'],
summary: {
estimatedServers: null,
cpuCoresAvailable: null,
memoryBytesAvailable: null,
portsAvailable: null,
eligibleHosts: 0,
totalHosts: 0,
startupSlots: null,
startupLimit: null,
activeStartups: null,
queued: null
}
};
const libraries = { t5: newestReadyLibraryId('t5'), t6: newestReadyLibraryId('t6') };
const requestedGames = query.game === 'all' ? ['t5', 't6'] as const : [query.game];
const requestedGames = query.game === 'all' ? (['t5', 't6'] as const) : [query.game];
const now = Date.now();
const hosts: CapacityHost[] = [...snapshot.hosts.values()].filter(({ host }) => !query.hostId || host.id === query.hostId).map(observation => {
const { host, calibration } = observation;
const fresh = !!host.lastHeartbeat && now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS;
const connected = host.connected && fresh;
const games = (['t5', 't6'] as const).filter(game => libraries[game] && host.inventory.libraries?.some(
library => library.id === libraries[game] && ['ready', 'outdated'].includes(library.state)));
const reasons: string[] = [];
if (!connected) reasons.push(host.enrolled ? 'Waiting for a fresh host report.' : 'This host has not enrolled yet.');
if (host.mode !== 'active') reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.');
if (host.decommission) reasons.push('Host is being removed.');
if (!host.credentialActive && host.enrolled) reasons.push('Host credentials are inactive.');
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage)) reasons.push('The default runtime is not ready.');
if (!requestedGames.some(game => games.includes(game))) reasons.push('Latest game files for this selection are not ready.');
if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.');
const eligible = connected && host.readiness === 'ready' && reasons.length === 0;
const cpuCoresAvailable = fresh ? observation.freeCpu : null;
const memoryBytesAvailable = fresh ? observation.freeMemory : null;
const portsAvailable = freePorts(observation);
const limits = { cpu: cpuCoresAvailable === null ? null : Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)),
memory: memoryBytesAvailable === null ? null : Math.max(0, Math.floor(memoryBytesAvailable / calibration.running.memoryBytes)),
ports: portsAvailable };
const estimatedServers = !eligible ? 0 : limits.cpu === null || limits.memory === null ? null : Math.min(limits.cpu, limits.memory, limits.ports);
const limitingResource: CapacityHost['limitingResource'] = estimatedServers === null || !eligible ? null
: limits.ports === estimatedServers ? 'Ports' : limits.memory === estimatedServers ? 'Memory' : 'CPU';
if (eligible && (limits.cpu === null || limits.memory === null)) reasons.push('Waiting for valid CPU and memory measurements.');
else if (eligible && estimatedServers === 0) reasons.push(`No additional instances currently fit: ${limitingResource?.toLowerCase()} capacity is full.`);
return { id: host.id, name: host.name, readiness: host.readiness, connected, updatedAt: host.lastHeartbeat,
eligible, games, reasons: [...new Set(reasons)], cpuCoresAvailable, memoryBytesAvailable,
portsAvailable, totalPorts: host.portMax - host.portMin + 1, estimatedServers, limits, limitingResource, calibration };
}).sort((first, second) => first.name.localeCompare(second.name));
const eligible = hosts.filter(host => host.eligible);
const hosts: CapacityHost[] = [...snapshot.hosts.values()]
.filter(({ host }) => !query.hostId || host.id === query.hostId)
.map((observation) => {
const { host, calibration } = observation;
const fresh =
!!host.lastHeartbeat &&
now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS;
const connected = host.connected && fresh;
const games = (['t5', 't6'] as const).filter(
(game) =>
libraries[game] &&
host.inventory.libraries?.some(
(library) =>
library.id === libraries[game] &&
['ready', 'outdated'].includes(library.state)
)
);
const reasons: string[] = [];
if (!connected)
reasons.push(
host.enrolled
? 'Waiting for a fresh host report.'
: 'This host has not enrolled yet.'
);
if (host.mode !== 'active')
reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.');
if (host.decommission) reasons.push('Host is being removed.');
if (!host.credentialActive && host.enrolled)
reasons.push('Host credentials are inactive.');
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage))
reasons.push('The default runtime is not ready.');
if (!requestedGames.some((game) => games.includes(game)))
reasons.push('Latest game files for this selection are not ready.');
if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3)
reasons.push('Storage is below the launch reserve.');
const eligible = connected && host.readiness === 'ready' && reasons.length === 0;
const cpuCoresAvailable = fresh ? observation.freeCpu : null;
const memoryBytesAvailable = fresh ? observation.freeMemory : null;
const portsAvailable = freePorts(observation);
const limits = {
cpu:
cpuCoresAvailable === null
? null
: Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)),
memory:
memoryBytesAvailable === null
? null
: Math.max(
0,
Math.floor(memoryBytesAvailable / calibration.running.memoryBytes)
),
ports: portsAvailable
};
const estimatedServers = !eligible
? 0
: limits.cpu === null || limits.memory === null
? null
: Math.min(limits.cpu, limits.memory, limits.ports);
const limitingResource: CapacityHost['limitingResource'] =
estimatedServers === null || !eligible
? null
: limits.ports === estimatedServers
? 'Ports'
: limits.memory === estimatedServers
? 'Memory'
: 'CPU';
if (eligible && (limits.cpu === null || limits.memory === null))
reasons.push('Waiting for valid CPU and memory measurements.');
else if (eligible && estimatedServers === 0)
reasons.push(
`No additional instances currently fit: ${limitingResource?.toLowerCase()} capacity is full.`
);
return {
id: host.id,
name: host.name,
readiness: host.readiness,
connected,
updatedAt: host.lastHeartbeat,
eligible,
games,
reasons: [...new Set(reasons)],
cpuCoresAvailable,
memoryBytesAvailable,
portsAvailable,
totalPorts: host.portMax - host.portMin + 1,
estimatedServers,
limits,
limitingResource,
calibration
};
})
.sort((first, second) => first.name.localeCompare(second.name));
const eligible = hosts.filter((host) => host.eligible);
const sum = (key: 'cpuCoresAvailable' | 'memoryBytesAvailable' | 'estimatedServers') =>
eligible.some(host => host[key] === null) ? null : eligible.reduce((total, host) => total + (host[key] ?? 0), 0);
eligible.some((host) => host[key] === null)
? null
: eligible.reduce((total, host) => total + (host[key] ?? 0), 0);
const warnings: string[] = [];
if (!hosts.length) warnings.push(query.hostId ? 'The selected host is unavailable.' : 'Add a host to start tracking launch capacity.');
else if (!eligible.length) warnings.push('No host is ready for this game selection. Review the host details below.');
else if (eligible.length < hosts.length) warnings.push(`${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.`);
if (eligible.some(host => host.estimatedServers === null)) warnings.push('Some resource measurements are missing; the total estimate is unknown.');
if (!requestedGames.some(game => libraries[game])) warnings.push('Upload game files before launching this selection.');
return { updatedAt: snapshot.updatedAt, hosts,
hostOptions: [...snapshot.hosts.values()].map(({ host }) => ({ id: host.id, name: host.name })).sort((a, b) => a.name.localeCompare(b.name)),
warnings, summary: { estimatedServers: sum('estimatedServers'), cpuCoresAvailable: sum('cpuCoresAvailable'),
memoryBytesAvailable: sum('memoryBytesAvailable'), portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0),
eligibleHosts: eligible.length, totalHosts: hosts.length, startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active),
startupLimit: snapshot.startupLimit, activeStartups: snapshot.active, queued: snapshot.queued } };
if (!hosts.length)
warnings.push(
query.hostId
? 'The selected host is unavailable.'
: 'Add a host to start tracking launch capacity.'
);
else if (!eligible.length)
warnings.push('No host is ready for this game selection. Review the host details below.');
else if (eligible.length < hosts.length)
warnings.push(
`${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.`
);
if (eligible.some((host) => host.estimatedServers === null))
warnings.push('Some resource measurements are missing; the total estimate is unknown.');
if (!requestedGames.some((game) => libraries[game]))
warnings.push('Upload game files before launching this selection.');
return {
updatedAt: snapshot.updatedAt,
hosts,
hostOptions: [...snapshot.hosts.values()]
.map(({ host }) => ({ id: host.id, name: host.name }))
.sort((a, b) => a.name.localeCompare(b.name)),
warnings,
summary: {
estimatedServers: sum('estimatedServers'),
cpuCoresAvailable: sum('cpuCoresAvailable'),
memoryBytesAvailable: sum('memoryBytesAvailable'),
portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0),
eligibleHosts: eligible.length,
totalHosts: hosts.length,
startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active),
startupLimit: snapshot.startupLimit,
activeStartups: snapshot.active,
queued: snapshot.queued
}
};
}
+2 -1
View File
@@ -5,6 +5,7 @@ import type { CapacityData, CapacityQuery } from '../../capacity.js';
export async function capacityPage(query: CapacityQuery): Promise<CapacityData> {
const current = await readCapacityCurrent(query);
if (query.hostId && !current.hostOptions.some(host => host.id === query.hostId)) error(404, 'Host not found.');
if (query.hostId && !current.hostOptions.some((host) => host.id === query.hostId))
error(404, 'Host not found.');
return { query, current, history: await capacityHistoryReport(query) };
}
+94 -29
View File
@@ -10,9 +10,13 @@ const MEMORY_HEADROOM = 500_000_000;
const CPU_USABLE_FRACTION = 0.95;
export class CapacityError extends Error {}
export interface PlacementOptions {
hostId?: string; libraryId: string; image: string; bindAddress?: string;
hostId?: string;
libraryId: string;
image: string;
bindAddress?: string;
excludeHostIds?: string[];
thorough?: boolean; excludeInstanceId?: string;
thorough?: boolean;
excludeInstanceId?: string;
// Execution only: not-yet-submitted launches wait their turn instead of
// requiring their entire backlog to fit before any one can start.
dispatch?: boolean;
@@ -20,12 +24,24 @@ export interface PlacementOptions {
}
/** One indexed port read per host, including both protocols and wildcard binds. */
export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}): number | null {
export function availableHostPort(
host: Host,
options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}
): number | null {
const bind = options.bindAddress ?? host.bindAddress;
const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
const rows = db()
.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
.all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[];
const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind)
|| ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port));
const used = new Set(
rows
.filter(
(row) =>
row.bind_address === bind ||
['0.0.0.0', '::'].includes(bind) ||
['0.0.0.0', '::'].includes(row.bind_address)
)
.map((row) => row.port)
);
for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port);
for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port;
return null;
@@ -33,38 +49,66 @@ export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'b
function projectedLoad(host: Host, options: PlacementOptions): number {
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) {
if (
!cpuCount ||
cpuUsagePercent === undefined ||
!memoryTotal ||
memoryAvailable === undefined ||
memoryAvailable > memoryTotal
) {
throw new Error('Waiting for valid CPU and memory measurements.');
}
let historicalCpu = cpuUsagePercent;
let historicalAvailable = memoryAvailable;
if (options.thorough) {
const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`)
.get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null };
const history = db()
.prepare(
`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`
)
.get(
host.id,
new Date(Date.now() - 5 * 60_000).toISOString(),
cpuCount,
memoryTotal
) as { cpu: number | null; memory: number | null };
historicalCpu = Math.max(cpuUsagePercent, history.cpu ?? cpuUsagePercent);
historicalAvailable = Math.min(memoryAvailable, history.memory ?? memoryAvailable);
}
const instances = db().prepare(`SELECT i.id, i.generation,
const instances = db()
.prepare(
`SELECT i.id, i.generation,
NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE n.instance_id=i.id AND n.generation=i.generation AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE n.instance_id=i.id AND d.action='create' AND d.state='queued' AND d.submitted_at IS NULL
AND n.operation_id IS NULL AND n.stage_operation_id IS NULL) AS unsubmitted
FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
.all(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as
{ id: string; generation: number; pending: number; unsubmitted: number }[];
FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`
)
.all(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as {
id: string;
generation: number;
pending: number;
unsubmitted: number;
}[];
const { running, startup } = capacityCalibration(host.id);
const incoming = db().prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations
WHERE target_host_id=? AND removed_at IS NULL AND instance_id!=?`).get(host.id, options.excludeInstanceId ?? '') as { n: number };
const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
const incoming = db()
.prepare(
`SELECT COUNT(*) AS n FROM ec_host_migrations
WHERE target_host_id=? AND removed_at IS NULL AND instance_id!=?`
)
.get(host.id, options.excludeInstanceId ?? '') as { n: number };
const planned = options.planned?.filter((item) => item.hostId === host.id).length ?? 0;
// Admission budgets the eventual running collection. Dispatch budgets only
// the next startup; the rest of the queue waits for live capacity to recover.
const launch = options.dispatch ? startup : running;
let extraCpu = planned * running.cpuCores + launch.cpuCores + incoming.n * startup.cpuCores;
let extraMemory = planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes;
let extraMemory =
planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes;
let reservations = planned + 1 + incoming.n;
const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
const measurements = new Map(
host.inventory.instanceResources?.map((sample) => [sample.instanceId, sample])
);
for (const instance of instances) {
if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue;
if (instance.unsubmitted) {
@@ -82,28 +126,45 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
extraMemory += startup.memoryBytes;
reservations++;
}
const cores = cpuCount * cpuUsagePercent / 100 + extraCpu;
const cores = (cpuCount * cpuUsagePercent) / 100 + extraCpu;
const usedMemory = memoryTotal - memoryAvailable;
const memory = usedMemory + extraMemory;
const usableMemory = memoryTotal - MEMORY_HEADROOM;
const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`);
if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. `
+ `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).`);
if (cores > cpuCount * CPU_USABLE_FRACTION)
throw new Error(
`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`
);
if (memory > usableMemory)
throw new Error(
`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. ` +
`${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).`
);
// Recovered resources must not remain unavailable until the historical
// average catches up. History ranks hosts after current capacity passes.
const historicalMemory = memoryTotal - historicalAvailable + extraMemory;
const historicalCores = cpuCount * historicalCpu / 100 + extraCpu;
const historicalCores = (cpuCount * historicalCpu) / 100 + extraCpu;
return Math.max(historicalCores / cpuCount, historicalMemory / memoryTotal);
}
/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */
export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } {
export function choosePlacement(options: PlacementOptions): {
host: Host;
port: number;
load: number;
} {
const candidates: { host: Host; port: number; load: number }[] = [];
const reasons: string[] = [];
for (const host of listHosts().filter(host => (!options.hostId || host.id === options.hostId) && !options.excludeHostIds?.includes(host.id))) {
for (const host of listHosts().filter(
(host) =>
(!options.hostId || host.id === options.hostId) &&
!options.excludeHostIds?.includes(host.id)
)) {
try {
requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
requireReadyHost(host.id, {
image: options.image,
requiredBytes: INSTANCE_STORAGE_BYTES
});
assertReadyLibrary(host.id, options.libraryId);
const port = availableHostPort(host, options);
if (port === null) throw new Error('No available UDP/TCP game port.');
@@ -111,9 +172,13 @@ export function choosePlacement(options: PlacementOptions): { host: Host; port:
const candidate = { host, port, load };
if (!options.thorough) return candidate;
candidates.push(candidate);
} catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); }
} catch (error) {
reasons.push(`${host.name}: ${(error as Error).message}`);
}
}
candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id));
if (candidates.length) return candidates[0];
throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`);
throw new CapacityError(
`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`
);
}
+30 -9
View File
@@ -6,14 +6,23 @@ const CHUNK_CHARACTERS = 16 * 1024;
const CHUNK_BYTES = 64 * 1024;
const encoder = new TextEncoder();
function* jsonParts(operations: OperationDeliveryRow[], serverTime: string): Generator<string | Buffer> {
function* jsonParts(
operations: OperationDeliveryRow[],
serverTime: string
): Generator<string | Buffer> {
yield '{"operations":[';
for (const [index, operation] of operations.entries()) {
if (index) yield ',';
const metadata = JSON.stringify({
id: operation.id, hostId: operation.host_id, kind: operation.kind,
instanceId: operation.instance_id, generation: operation.generation, state: operation.state,
error: operation.error, createdAt: operation.created_at, updatedAt: operation.updated_at
id: operation.id,
hostId: operation.host_id,
kind: operation.kind,
instanceId: operation.instance_id,
generation: operation.generation,
state: operation.state,
error: operation.error,
createdAt: operation.created_at,
updatedAt: operation.updated_at
});
yield metadata.slice(0, -1) + ',"payload":';
// Load only the immutable payload currently being sent, after the
@@ -26,10 +35,14 @@ function* jsonParts(operations: OperationDeliveryRow[], serverTime: string): Gen
yield '],"serverTime":' + JSON.stringify(serverTime) + '}';
}
function* encodedChunks(operations: OperationDeliveryRow[], serverTime: string): Generator<Uint8Array> {
function* encodedChunks(
operations: OperationDeliveryRow[],
serverTime: string
): Generator<Uint8Array> {
for (const part of jsonParts(operations, serverTime)) {
if (Buffer.isBuffer(part)) {
for (let start = 0; start < part.length; start += CHUNK_BYTES) yield part.subarray(start, start + CHUNK_BYTES);
for (let start = 0; start < part.length; start += CHUNK_BYTES)
yield part.subarray(start, start + CHUNK_BYTES);
continue;
}
for (let start = 0; start < part.length;) {
@@ -45,7 +58,10 @@ function* encodedChunks(operations: OperationDeliveryRow[], serverTime: string):
}
/** Bound encoding work and yield between chunks so health requests keep flowing. */
export function hostOperationsResponse(operations: OperationDeliveryRow[], serverTime = new Date().toISOString()): Response {
export function hostOperationsResponse(
operations: OperationDeliveryRow[],
serverTime = new Date().toISOString()
): Response {
const chunks = encodedChunks(operations, serverTime);
let cancelled = false;
const stream = new ReadableStream<Uint8Array>({
@@ -56,7 +72,12 @@ export function hostOperationsResponse(operations: OperationDeliveryRow[], serve
if (next.done) controller.close();
else controller.enqueue(next.value);
},
cancel() { cancelled = true; chunks.return(undefined); }
cancel() {
cancelled = true;
chunks.return(undefined);
}
});
return new Response(stream, {
headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' }
});
return new Response(stream, { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' } });
}
+15 -3
View File
@@ -6,7 +6,12 @@ const visible = (alias: string) => `${alias}.history_expired_at IS NULL`;
/** Durable totals for global, host, server, and host/server filters; receipts update only changed membership. */
export function installHistoryCounts(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM ec_host_schema_migrations WHERE name='history-counts-v1'").get()) return;
if (
handle
.prepare("SELECT 1 FROM ec_host_schema_migrations WHERE name='history-counts-v1'")
.get()
)
return;
handle.transaction(() => {
handle.exec(`CREATE TABLE ec_host_operation_history_counts (
host_id TEXT NOT NULL, server_name TEXT NOT NULL, total INTEGER NOT NULL,
@@ -15,7 +20,12 @@ export function installHistoryCounts(handle: Database.Database): void {
DROP INDEX ec_host_operations_important_history;
CREATE INDEX ec_host_operations_important_history ON ec_host_operations(created_at)
WHERE history_expired_at IS NULL AND ${importantOperationSql.replaceAll('o.', '')};`);
for (const [host, server] of [['', ''], ['o.host_id', ''], ['', 'o.server_name'], ['o.host_id', 'o.server_name']]) {
for (const [host, server] of [
['', ''],
['o.host_id', ''],
['', 'o.server_name'],
['o.host_id', 'o.server_name']
]) {
const groups = [host, server].filter(Boolean);
handle.exec(`INSERT INTO ec_host_operation_history_counts
SELECT ${host || "''"},${server || "''"},COUNT(*),COALESCE(SUM(o.routine=1),0),COALESCE(SUM(${important('o')}),0)
@@ -37,6 +47,8 @@ export function installHistoryCounts(handle: Database.Database): void {
WHEN OLD.host_id IS NOT NEW.host_id OR OLD.server_name IS NOT NEW.server_name OR OLD.routine IS NOT NEW.routine
OR (${visible('OLD')}) IS NOT (${visible('NEW')}) OR ${important('OLD')} IS NOT ${important('NEW')}
BEGIN ${adjust('OLD', -1)} ${adjust('NEW', 1)} END;`);
handle.prepare("INSERT INTO ec_host_schema_migrations(name) VALUES('history-counts-v1')").run();
handle
.prepare("INSERT INTO ec_host_schema_migrations(name) VALUES('history-counts-v1')")
.run();
})();
}
+35 -11
View File
@@ -3,7 +3,8 @@ import type { Host } from '../../hosts.js';
import { authenticateCredential } from './index.js';
export function requireAgentTransport(request: Request): void {
if (new URL(request.url).protocol !== 'https:' && process.env.EC_AGENT_ALLOW_HTTP !== 'true') error(426, 'Host agents require HTTPS.');
if (new URL(request.url).protocol !== 'https:' && process.env.EC_AGENT_ALLOW_HTTP !== 'true')
error(426, 'Host agents require HTTPS.');
}
export function authenticateAgent(request: Request): Host {
requireAgentTransport(request);
@@ -15,33 +16,56 @@ export function authenticateAgent(request: Request): Host {
}
export function requireManagementOrigin(request: Request): void {
const origin = request.headers.get('origin');
if ((origin && origin !== new URL(request.url).origin) || request.headers.get('sec-fetch-site') === 'cross-site') error(403, 'Cross-origin host management requests are not allowed.');
if (
(origin && origin !== new URL(request.url).origin) ||
request.headers.get('sec-fetch-site') === 'cross-site'
)
error(403, 'Cross-origin host management requests are not allowed.');
// Agent credentials only authorize agent endpoints, never host management.
if (request.headers.get('authorization')?.match(/^Bearer ec_(?:host|enroll)_/i)) error(403, 'Agent credentials cannot manage the controller.');
if (request.headers.get('authorization')?.match(/^Bearer ec_(?:host|enroll)_/i))
error(403, 'Agent credentials cannot manage the controller.');
}
export async function objectBody(request: Request, maxBytes = 1024 * 1024): Promise<Record<string, unknown>> {
if (Number(request.headers.get('content-length')) > maxBytes) error(413, 'Request is too large.');
export async function objectBody(
request: Request,
maxBytes = 1024 * 1024
): Promise<Record<string, unknown>> {
if (Number(request.headers.get('content-length')) > maxBytes)
error(413, 'Request is too large.');
const reader = request.body?.getReader();
if (!reader) error(400, 'Request body must be valid JSON.');
const chunks: Uint8Array[] = []; let received = 0;
const chunks: Uint8Array[] = [];
let received = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
received += value.byteLength;
if (received > maxBytes) { void reader.cancel().catch(() => {}); error(413, 'Request is too large.'); }
if (received > maxBytes) {
void reader.cancel().catch(() => {});
error(413, 'Request is too large.');
}
chunks.push(value);
}
} finally { reader.releaseLock(); }
} finally {
reader.releaseLock();
}
const raw = Buffer.concat(chunks, received).toString('utf8');
let body: unknown;
try { body = JSON.parse(raw); } catch { error(400, 'Request body must be valid JSON.'); }
if (!body || typeof body !== 'object' || Array.isArray(body)) error(400, 'Request body must be an object.');
try {
body = JSON.parse(raw);
} catch {
error(400, 'Request body must be valid JSON.');
}
if (!body || typeof body !== 'object' || Array.isArray(body))
error(400, 'Request body must be an object.');
return body as Record<string, unknown>;
}
export function hostApiError(cause: unknown): Response {
if (cause && typeof cause === 'object' && 'status' in cause && 'body' in cause) throw cause;
return json({ error: cause instanceof Error ? cause.message : 'Host request failed.' }, { status: 400 });
return json(
{ error: cause instanceof Error ? cause.message : 'Host request failed.' },
{ status: 400 }
);
}
export function installerCommand(controller: string, token: string): string {
const origin = new URL(controller).origin;
File diff suppressed because it is too large Load Diff
+6 -2
View File
@@ -5,10 +5,14 @@ import { observationReader } from '../fleet/status.js';
/** Current instances remain visible when their host or game stops reporting. */
export function hostInstances(hostId: string): ServerObservation[] {
const servers = db().prepare(`SELECT s.* FROM servers s
const servers = db()
.prepare(
`SELECT s.* FROM servers s
JOIN native_instances i ON i.server = s.name
WHERE i.host_id = ? AND i.retired_at IS NULL AND s.destroyed_at IS NULL
ORDER BY s.name`).all(hostId) as ServerRow[];
ORDER BY s.name`
)
.all(hostId) as ServerRow[];
if (!servers.length) return [];
const observe = observationReader();
return servers.map(observe);
+3 -1
View File
@@ -9,6 +9,8 @@ export function logResultMetadata(value: unknown): Record<string, unknown> | nul
...(typeof result.truncated === 'boolean' ? { truncated: result.truncated } : {}),
...(typeof result.reset === 'boolean' ? { reset: result.reset } : {}),
...(Array.isArray(result.lines) ? { lineCount: result.lines.length } : {}),
...(typeof result.contentBase64 === 'string' ? { bytesRead: Buffer.byteLength(result.contentBase64, 'base64') } : {})
...(typeof result.contentBase64 === 'string'
? { bytesRead: Buffer.byteLength(result.contentBase64, 'base64') }
: {})
};
}
+37 -9
View File
@@ -1,23 +1,50 @@
export const observationKinds = ['host.inspect', 'instance.inspect', 'rcon.status-batch', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read'];
export const observationKinds = [
'host.inspect',
'instance.inspect',
'rcon.status-batch',
'files.read',
'files.list',
'files.stat',
'files.manifest',
'logs.read'
];
const commands = ['status', 'getstatus', 'version'];
export function safeOperationTarget(kind: string, value: string | null): string | null {
if (!value || value.length > 1024) return null;
const paths = kind === 'files.manifest' ? value.split(', ') : [value];
return paths.every(path => path === '.' || !/[\\\x00-\x1f\x7f]/.test(path)
&& path.split('/').every(part => part !== '' && part !== '.' && part !== '..')) ? value : null;
return paths.every(
(path) =>
path === '.' ||
// eslint-disable-next-line no-control-regex -- Reject control characters in observed paths.
(!/[\\\x00-\x1f\x7f]/.test(path) &&
path.split('/').every((part) => part !== '' && part !== '.' && part !== '..'))
)
? value
: null;
}
export function successfulObservation(kind: string, payload: Record<string, unknown>, result: unknown): boolean {
const routine = observationKinds.includes(kind) || kind === 'rcon.execute' && commands.includes(String(payload.command));
const value = result && typeof result === 'object' ? result as Record<string, unknown> : {};
export function successfulObservation(
kind: string,
payload: Record<string, unknown>,
result: unknown
): boolean {
const routine =
observationKinds.includes(kind) ||
(kind === 'rcon.execute' && commands.includes(String(payload.command)));
const value = result && typeof result === 'object' ? (result as Record<string, unknown>) : {};
// Older agents set ok=false when any target fails, despite completing the batch.
const completedBatch = kind === 'rcon.status-batch' && Array.isArray(value.results);
return routine && (completedBatch || value.ok !== false) && value.healthy !== false && (value.error == null || value.error === '');
return (
routine &&
(completedBatch || value.ok !== false) &&
value.healthy !== false &&
(value.error == null || value.error === '')
);
}
/** Uses the operation alias o; serialized payloads may live outside the routing row. */
export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')}) THEN 1
export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map((kind) => `'${kind}'`).join(',')}) THEN 1
WHEN o.kind='rcon.execute' THEN json_extract(COALESCE(
(SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')
ELSE 0 END)`;
@@ -32,7 +59,8 @@ export const successfulObservationSql = `(o.state='completed' AND COALESCE(o.err
export const importantOperationSql = `(o.routine=0 OR o.state IN ('failed','unknown')
OR (o.state='completed' AND o.observation_success=0) OR COALESCE(o.error,'')!='')`;
const payloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
const payloadSql =
'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
/** Deliberately exclude arbitrary RCON commands and all file contents/credentials. */
export const operationTargetSql = `CASE
WHEN o.kind='plutonium.prepare' THEN CASE
+26 -7
View File
@@ -5,25 +5,44 @@ import { observationReader } from '../fleet/status.js';
import type { ServerRow } from '../fleet/servers.js';
import type { Host } from '../../hosts.js';
export interface HostOverview extends Host { serverCount: number; playerCount: number; unreportedInstances: number }
export interface HostOverview extends Host {
serverCount: number;
playerCount: number;
unreportedInstances: number;
}
/** Read current placements once and use the fleet's shared telemetry/RCON selection. */
export function hostOverview(): HostOverview[] {
const hosts = listHosts();
const instances = db().prepare(`SELECT s.*, i.host_id FROM native_instances i
const instances = db()
.prepare(
`SELECT s.*, i.host_id FROM native_instances i
JOIN servers s ON s.name = i.server
WHERE i.retired_at IS NULL AND s.destroyed_at IS NULL`).all() as (ServerRow & { host_id: string })[];
const counts = new Map<string, { serverCount: number; playerCount: number; unreportedInstances: number }>();
WHERE i.retired_at IS NULL AND s.destroyed_at IS NULL`
)
.all() as (ServerRow & { host_id: string })[];
const counts = new Map<
string,
{ serverCount: number; playerCount: number; unreportedInstances: number }
>();
if (instances.length) {
const observe = observationReader();
for (const instance of instances) {
const count = counts.get(instance.host_id) ?? { serverCount: 0, playerCount: 0, unreportedInstances: 0 };
const count = counts.get(instance.host_id) ?? {
serverCount: 0,
playerCount: 0,
unreportedInstances: 0
};
count.serverCount++;
const observation = observe(instance);
if (observation.health === 'online' && observation.last_players !== null) count.playerCount += observation.last_players;
if (observation.health === 'online' && observation.last_players !== null)
count.playerCount += observation.last_players;
else count.unreportedInstances++;
counts.set(instance.host_id, count);
}
}
return hosts.map(host => ({ ...presentHostPreparation(host), ...(counts.get(host.id) ?? { serverCount: 0, playerCount: 0, unreportedInstances: 0 }) }));
return hosts.map((host) => ({
...presentHostPreparation(host),
...(counts.get(host.id) ?? { serverCount: 0, playerCount: 0, unreportedInstances: 0 })
}));
}
+329 -94
View File
@@ -1,25 +1,65 @@
import { db } from '../db/index.js';
import { config } from '../config.js';
import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation, shouldDeferPlutoniumPreparation } from './index.js';
import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
import {
AGENT_PROTOCOL_VERSION,
getHost,
queueHostOperation,
shouldDeferPlutoniumPreparation
} from './index.js';
import {
getHostLibraryCache,
latestPublishedLibraries,
queueLibraryDownload
} from '../game-files/index.js';
import { GAME_FILE_GAMES } from '../../game-files.js';
import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
import type {
Host,
HostOperationState,
HostPreparation,
HostPreparationItem
} from '../../hosts.js';
import { scheduleAvailabilityRefresh } from './availability.js';
import { readSetting } from '../maintenance/settings.js';
import type { PlutoniumRelease } from '../../plutonium.js';
const activeStates = ['pending', 'accepted', 'running', 'unknown'];
const supersededDownload = 'Superseded by newer game files before delivery to the host.';
type PreparationOperation = { id: string; sequence: number; last_delivered_at: string | null; state: HostOperationState; updated_at: string; error: string | null };
type Target = { item: HostPreparationItem; image?: string; versionId?: string; revision?: number; requiredBytes: number; operation?: PreparationOperation };
type PreparationOperation = {
id: string;
sequence: number;
last_delivered_at: string | null;
state: HostOperationState;
updated_at: string;
error: string | null;
};
type Target = {
item: HostPreparationItem;
image?: string;
versionId?: string;
revision?: number;
requiredBytes: number;
operation?: PreparationOperation;
};
type QueuedSelection = { libraryId: string };
type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
type LibraryTarget = Pick<
ReturnType<typeof latestPublishedLibraries>[number],
'id' | 'sizeBytes' | 'manifestSha256'
> & { name: string; queued?: boolean };
/** Accepted requests retain their prerequisites until placement owns an instance. */
function queuedSelections(hostId: string): QueuedSelection[] {
// Standalone host/library readers can initialize before provisioning migrations.
if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'").get()) return [];
return db().prepare(`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId FROM native_creation_intents q
if (
!db()
.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'"
)
.get()
)
return [];
return db()
.prepare(
`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId FROM native_creation_intents q
JOIN server_deployments d ON d.id=q.deployment_id
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL
@@ -30,169 +70,337 @@ function queuedSelections(hostId: string): QueuedSelection[] {
JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
AND (m.target_host_id IS NULL OR m.target_host_id=?)`).all(hostId, hostId) as QueuedSelection[];
AND (m.target_host_id IS NULL OR m.target_host_id=?)`
)
.all(hostId, hostId) as QueuedSelection[];
}
function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
const latest = latestPublishedLibraries();
const latestIds = new Set(latest.map(version => version.id));
const latestIds = new Set(latest.map((version) => version.id));
const queued = new Map<string, LibraryTarget>();
const get = db().prepare(`SELECT v.id,l.game,v.version,v.size_bytes AS sizeBytes,v.manifest_sha256 AS manifestSha256
const get = db()
.prepare(`SELECT v.id,l.game,v.version,v.size_bytes AS sizeBytes,v.manifest_sha256 AS manifestSha256
FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
WHERE v.id=? AND v.state='ready' AND NOT EXISTS(SELECT 1 FROM game_file_cleanup c WHERE c.version_id=v.id)`);
for (const selection of selections) {
if (latestIds.has(selection.libraryId) || queued.has(selection.libraryId)) continue;
const version = get.get(selection.libraryId) as { id: string; game: string; version: string; sizeBytes: number; manifestSha256: string } | undefined;
if (version) queued.set(version.id, { ...version, queued: true,
name: `${GAME_FILE_GAMES.find(game => game.id === version.game)?.name ?? version.game} · Queued version ${version.version}` });
const version = get.get(selection.libraryId) as
| {
id: string;
game: string;
version: string;
sizeBytes: number;
manifestSha256: string;
}
| undefined;
if (version)
queued.set(version.id, {
...version,
queued: true,
name: `${GAME_FILE_GAMES.find((game) => game.id === version.game)?.name ?? version.game} · Queued version ${version.version}`
});
}
// Finish prerequisites for accepted work before downloading newer optional files.
return [...queued.values(), ...latest];
}
function latestOperation(hostId: string, kind: string, field?: string, value?: string): PreparationOperation | undefined {
return db().prepare(`SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations
WHERE host_id=? AND kind=? ${field ? 'AND json_extract(payload,?)=?' : ''} ORDER BY created_at DESC,rowid DESC LIMIT 1`)
.get(hostId, kind, ...(field ? [`$.${field}`, value!] : [])) as PreparationOperation | undefined;
function latestOperation(
hostId: string,
kind: string,
field?: string,
value?: string
): PreparationOperation | undefined {
return db()
.prepare(
`SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations
WHERE host_id=? AND kind=? ${field ? 'AND json_extract(payload,?)=?' : ''} ORDER BY created_at DESC,rowid DESC LIMIT 1`
)
.get(hostId, kind, ...(field ? [`$.${field}`, value!] : [])) as
PreparationOperation | undefined;
}
function retryAt(hostId: string, target: Target): number {
const operation = target.operation;
if (!operation || operation.state !== 'failed') return 0;
const kind = target.image ? 'runtime.pull' : target.versionId ? 'library.download' : 'plutonium.prepare';
const kind = target.image
? 'runtime.pull'
: target.versionId
? 'library.download'
: 'plutonium.prepare';
const field = target.image ? '$.image' : target.versionId ? '$.libraryVersionId' : null;
const value = target.image ?? target.versionId;
const match = field ? 'AND json_extract(payload,?)=?' : '';
const bindings = [hostId, kind, ...(field ? [field, value!] : [])];
// Count consecutive failures, including across controller restarts. Unknown
// operations are never retried under a new ID: their journal must reconcile.
const { count } = db().prepare(`SELECT COUNT(*) AS count FROM ec_host_operations
const { count } = db()
.prepare(
`SELECT COUNT(*) AS count FROM ec_host_operations
WHERE host_id=? AND kind=? ${match} AND state='failed'
AND rowid>COALESCE((SELECT MAX(rowid) FROM ec_host_operations
WHERE host_id=? AND kind=? ${match} AND state='completed'),0)`)
WHERE host_id=? AND kind=? ${match} AND state='completed'),0)`
)
.get(...bindings, ...bindings) as { count: number };
return Date.parse(operation.updated_at) + Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1)));
return (
Date.parse(operation.updated_at) +
Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1)))
);
}
function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null {
if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' };
if (host.mode !== 'active' && !host.enablingSince) return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' };
if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' };
if (!host.inventory.dockerRunning) return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' };
if (!host.inventory.directoriesWritable) return { state: 'blocked', detail: 'The host cannot write its managed directories.' };
if (host.decommission)
return {
state: 'waiting',
detail: 'Automatic preparation is paused during host deletion.'
};
if (host.mode !== 'active' && !host.enablingSince)
return {
state: 'waiting',
detail: `Automatic preparation is paused while this host is ${host.mode}.`
};
if (!host.enrolled)
return {
state: 'waiting',
detail: 'Automatic preparation starts after agent installation.'
};
if (!host.connected || !host.credentialActive)
return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION)
return { state: 'blocked', detail: 'A compatible host agent is required.' };
if (!host.inventory.dockerRunning)
return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' };
if (!host.inventory.directoriesWritable)
return { state: 'blocked', detail: 'The host cannot write its managed directories.' };
return null;
}
function targets(host: Host): Target[] {
const enabling = db().prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?').get(host.id) as { operation_floor: number } | undefined;
const verifiedThisEnable = (operation: PreparationOperation | undefined) => !enabling || !!operation
&& operation.sequence > enabling.operation_floor && operation.state === 'completed'
&& Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at);
const enabling = db()
.prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?')
.get(host.id) as { operation_floor: number } | undefined;
const verifiedThisEnable = (operation: PreparationOperation | undefined) =>
!enabling ||
(!!operation &&
operation.sequence > enabling.operation_floor &&
operation.state === 'completed' &&
Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at));
const selections = queuedSelections(host.id);
// Queued launches and migrations are not authority to pull custom images.
const result: Target[] = [config.defaultRuntimeImage].map(image => {
const result: Target[] = [config.defaultRuntimeImage].map((image) => {
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
return { image, requiredBytes: host.minimumFreeBytes,
const runtimeReady =
host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
return {
image,
requiredBytes: host.minimumFreeBytes,
operation,
item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } };
item: {
kind: 'runtime',
label: image,
state: runtimeReady ? 'ready' : 'waiting',
detail: runtimeReady
? 'Runtime is cached.'
: 'Runtime will be downloaded automatically.'
}
};
});
for (const version of targetLibraries(selections)) {
const cache = getHostLibraryCache(host.id, version.id);
const matches = cache?.manifestSha256 === version.manifestSha256;
const operation = latestOperation(host.id, 'library.download', 'libraryVersionId', version.id);
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes && verifiedThisEnable(operation);
result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
const operation = latestOperation(
host.id,
'library.download',
'libraryVersionId',
version.id
);
const ready =
matches &&
['ready', 'outdated'].includes(cache!.status) &&
cache!.bytesDownloaded === version.sizeBytes &&
verifiedThisEnable(operation);
result.push({
versionId: version.id,
requiredBytes:
version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
operation,
item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting',
item: {
kind: 'library',
label: version.name,
state: ready ? 'ready' : 'waiting',
detail: version.queued
? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.'
: ready ? 'The latest game files are downloaded and verified.' : 'The latest game files will be downloaded and verified automatically.',
bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes, ...(cache ? { cache } : {}) } });
? ready
? 'Game files selected by queued launches are downloaded and verified.'
: 'Game files selected by queued launches will be downloaded and verified automatically.'
: ready
? 'The latest game files are downloaded and verified.'
: 'The latest game files will be downloaded and verified automatically.',
bytesDownloaded: cache?.bytesDownloaded ?? 0,
bytesTotal: version.sizeBytes,
...(cache ? { cache } : {})
}
});
}
if (host.inventory.capabilities?.includes('plutonium.cache')) {
// Read the same production release evidence as freshRelease without
// importing its fleet/provisioning observers into host preparation.
const release = readSetting<PlutoniumRelease>('plutonium_release', { revision: null, checkedAt: null, error: null });
const release = readSetting<PlutoniumRelease>('plutonium_release', {
revision: null,
checkedAt: null,
error: null
});
const age = release.checkedAt ? Date.now() - Date.parse(release.checkedAt) : NaN;
const revision = !release.error && age >= 0 && age < 60 * 60_000
&& Number.isSafeInteger(release.revision) && release.revision! > 0 ? release.revision! : 0;
const revision =
!release.error &&
age >= 0 &&
age < 60 * 60_000 &&
Number.isSafeInteger(release.revision) &&
release.revision! > 0
? release.revision!
: 0;
const cache = host.inventory.plutoniumCache;
const ready = !!cache && cache.revision >= revision;
result.push({ revision, requiredBytes: host.minimumFreeBytes,
result.push({
revision,
requiredBytes: host.minimumFreeBytes,
operation: latestOperation(host.id, 'plutonium.prepare'),
item: { kind: 'plutonium', label: 'Plutonium cache (optional)', state: ready ? 'ready' : 'waiting',
detail: ready ? `Production revision r${cache.revision} is cached for new deployments.`
: cache ? `Production revision r${revision} will replace cached r${cache.revision} automatically.`
: 'Plutonium files will be cached after the runtime and game files are ready.' } });
item: {
kind: 'plutonium',
label: 'Plutonium cache (optional)',
state: ready ? 'ready' : 'waiting',
detail: ready
? `Production revision r${cache.revision} is cached for new deployments.`
: cache
? `Production revision r${revision} will replace cached r${cache.revision} automatically.`
: 'Plutonium files will be cached after the runtime and game files are ready.'
}
});
}
const blocker = hostBlocker(host);
for (const target of result) {
if (target.item.state === 'ready') continue;
const operation = target.operation;
if (operation) target.item.operationId = operation.id;
if (blocker) { Object.assign(target.item, blocker); continue; }
if (blocker) {
Object.assign(target.item, blocker);
continue;
}
if (operation && activeStates.includes(operation.state)) {
target.item.state = 'preparing';
target.item.detail = target.versionId && operation.state === 'pending' && !operation.last_delivered_at
? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.'
: operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
: target.image ? 'Downloading the runtime automatically.'
: target.revision !== undefined ? 'Downloading and verifying the optional Plutonium cache.'
: target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
} else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
} else if ((target.revision !== undefined || !enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) {
target.item.detail =
target.versionId && operation.state === 'pending' && !operation.last_delivered_at
? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.'
: operation.state === 'unknown'
? 'Waiting for the agent to reconcile interrupted preparation.'
: target.image
? 'Downloading the runtime automatically.'
: target.revision !== undefined
? 'Downloading and verifying the optional Plutonium cache.'
: target.item.cache?.status === 'verifying'
? 'Download complete. Verifying all game files.'
: 'Downloading game files. Verification starts after all files arrive.';
} else if (
operation &&
Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)
) {
target.item.state = 'preparing';
target.item.detail = 'Waiting for a fresh host inventory after preparation.';
} else if (
(target.revision !== undefined ||
!enabling ||
(operation?.sequence ?? 0) > enabling.operation_floor) &&
retryAt(host.id, target) > Date.now()
) {
target.item.state = 'blocked';
target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
} else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
target.item.state = 'blocked'; target.item.detail = 'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.';
target.item.state = 'blocked';
target.item.detail =
'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.';
}
}
return result;
}
export function hostPreparationStatus(host: Host): HostPreparation {
const items = targets(host).map(target => target.item);
const items = targets(host).map((target) => target.item);
if (host.enablingSince) {
const published = latestPublishedLibraries();
for (const game of GAME_FILE_GAMES) if (!published.some(version => version.game === game.id)) {
items.push({ kind: 'library', label: game.name, state: 'blocked', detail: `Upload and publish ${game.name} game files before this host can be enabled.` });
}
} else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
detail: 'Upload Black Ops I or Black Ops II game files; hosts download the latest available files automatically.' });
for (const game of GAME_FILE_GAMES)
if (!published.some((version) => version.game === game.id)) {
items.push({
kind: 'library',
label: game.name,
state: 'blocked',
detail: `Upload and publish ${game.name} game files before this host can be enabled.`
});
}
} else if (!items.some((item) => item.kind === 'library'))
items.push({
kind: 'library',
label: 'Game files',
state: 'waiting',
detail: 'Upload Black Ops I or Black Ops II game files; hosts download the latest available files automatically.'
});
// Cache health is visible, but never makes a usable host ineligible or
// delays re-enabling: a lobby's normal updater handles cache misses.
const required = items.filter(item => item.kind !== 'plutonium');
const state = required.some(item => item.state === 'blocked') ? 'blocked'
: required.some(item => item.state === 'preparing') ? 'preparing'
: required.some(item => item.state === 'waiting') ? 'waiting' : 'ready';
const required = items.filter((item) => item.kind !== 'plutonium');
const state = required.some((item) => item.state === 'blocked')
? 'blocked'
: required.some((item) => item.state === 'preparing')
? 'preparing'
: required.some((item) => item.state === 'waiting')
? 'waiting'
: 'ready';
return { state, items };
}
export function presentHostPreparation(host: Host): Host {
const preparation = hostPreparationStatus(host);
if (['needs-runtime', 'needs-game-files'].includes(host.readiness) && preparation.state === 'preparing') {
return { ...host, readiness: 'preparing', readinessReasons: preparation.items.filter(item => item.kind !== 'plutonium' && item.state !== 'ready').map(item => item.detail), preparation };
if (
['needs-runtime', 'needs-game-files'].includes(host.readiness) &&
preparation.state === 'preparing'
) {
return {
...host,
readiness: 'preparing',
readinessReasons: preparation.items
.filter((item) => item.kind !== 'plutonium' && item.state !== 'ready')
.map((item) => item.detail),
preparation
};
}
return { ...host, preparation };
}
function supersedeUndeliveredDownloads(hostId: string): void {
const required = new Set(targetLibraries(queuedSelections(hostId)).map(version => version.id));
const pending = db().prepare(`SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId
const required = new Set(
targetLibraries(queuedSelections(hostId)).map((version) => version.id)
);
const pending = db()
.prepare(
`SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId
FROM ec_host_operations WHERE host_id=? AND kind='library.download' AND state='pending'
AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1`).all(hostId) as { id: string; versionId: string }[];
AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1`
)
.all(hostId) as { id: string; versionId: string }[];
const now = new Date().toISOString();
for (const operation of pending) {
if (required.has(operation.versionId)) continue;
// Delivered operations may already be writing files. Only unsubmitted
// automatic work can be replaced without consulting the agent journal.
db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(supersededDownload, now, operation.id);
db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)").run(operation.id, now, supersededDownload);
db().prepare("UPDATE host_library_caches SET status='missing',error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=? AND operation_id=? AND status NOT IN ('ready','outdated')")
db()
.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?")
.run(supersededDownload, now, operation.id);
db()
.prepare(
"INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)"
)
.run(operation.id, now, supersededDownload);
db()
.prepare(
"UPDATE host_library_caches SET status='missing',error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=? AND operation_id=? AND status NOT IN ('ready','outdated')"
)
.run(now, hostId, operation.versionId, operation.id);
}
}
@@ -207,28 +415,55 @@ export function reconcileHostPreparation(hostId: string): HostPreparation {
// Queue one required setup mutation at a time with fresh disk
// inventory. A running optional cache yields to required work via
// the heartbeat flag, then resumes under its original operation ID.
const active = db().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=?
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId);
const finished = db().prepare(`SELECT updated_at FROM ec_host_operations WHERE host_id=?
const active = db()
.prepare(
`SELECT 1 FROM ec_host_operations WHERE host_id=?
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`
)
.get(hostId);
const finished = db()
.prepare(
`SELECT updated_at FROM ec_host_operations WHERE host_id=?
AND kind IN ('runtime.pull','library.download','library.import','plutonium.prepare') AND state IN ('completed','failed')
AND (error IS NULL OR error != ?)
ORDER BY updated_at DESC LIMIT 1`).get(hostId, supersededDownload) as { updated_at: string } | undefined;
if (!active && (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))) {
ORDER BY updated_at DESC LIMIT 1`
)
.get(hostId, supersededDownload) as { updated_at: string } | undefined;
if (
!active &&
(!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))
) {
// A failed or oversized library must not prevent other libraries
// from preparing. The runtime still comes first when eligible.
const candidates = targets(host);
const requiredReady = candidates.every(target => target.item.kind === 'plutonium' || target.item.state === 'ready');
const target = candidates.find(target => target.item.state === 'waiting' && (target.item.kind !== 'plutonium' || requiredReady));
if (target?.image) queueHostOperation(host.id, 'runtime.pull', { image: target.image });
else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true });
else if (target?.revision !== undefined && !shouldDeferPlutoniumPreparation(host.id)) queueHostOperation(host.id, 'plutonium.prepare', {
revision: target.revision, minimumFreeBytes: host.minimumFreeBytes, automatic: true
});
const requiredReady = candidates.every(
(target) => target.item.kind === 'plutonium' || target.item.state === 'ready'
);
const target = candidates.find(
(target) =>
target.item.state === 'waiting' &&
(target.item.kind !== 'plutonium' || requiredReady)
);
if (target?.image)
queueHostOperation(host.id, 'runtime.pull', { image: target.image });
else if (target?.versionId)
queueLibraryDownload(host.id, target.versionId, { automatic: true });
else if (
target?.revision !== undefined &&
!shouldDeferPlutoniumPreparation(host.id)
)
queueHostOperation(host.id, 'plutonium.prepare', {
revision: target.revision,
minimumFreeBytes: host.minimumFreeBytes,
automatic: true
});
}
}
const status = hostPreparationStatus(host);
if (host.enablingSince && !hostBlocker(host) && status.state === 'ready') {
db().prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'").run(hostId);
db()
.prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'")
.run(hostId);
db().prepare('DELETE FROM ec_host_enable_requests WHERE host_id=?').run(hostId);
scheduleAvailabilityRefresh();
}
+8 -4
View File
@@ -2,11 +2,15 @@ import { readFile } from 'node:fs/promises';
import { resolve } from 'node:path';
/** adapter-node copies public assets to build/client; Vite serves static in development. */
export async function agentReleaseAsset(file: 'install.sh' | 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json'): Promise<string> {
export async function agentReleaseAsset(
file: 'install.sh' | 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json'
): Promise<string> {
// A prior production build must never shadow a freshly packaged development release.
if (process.env.NODE_ENV === 'development') return readFile(resolve('static/agent', file), 'utf8');
try { return await readFile(resolve('build/client/agent', file), 'utf8'); }
catch (cause) {
if (process.env.NODE_ENV === 'development')
return readFile(resolve('static/agent', file), 'utf8');
try {
return await readFile(resolve('build/client/agent', file), 'utf8');
} catch (cause) {
if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause;
return readFile(resolve('static/agent', file), 'utf8');
}
+3 -2
View File
@@ -7,8 +7,9 @@ const requests = new AsyncLocalStorage<{ inventories: InventoryCache | null }>()
export function withHostRequestContext<T>(work: () => T | Promise<T>): Promise<T> {
const context = { inventories: new Map() as InventoryCache | null };
return requests.run(context, async () => {
try { return await work(); }
finally {
try {
return await work();
} finally {
// Detached work can inherit this context. Release request data and
// stop caching when the response is ready, even if that work continues.
context.inventories = null;
+97 -31
View File
@@ -1,16 +1,29 @@
import { createHash } from 'node:crypto';
import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js';
import {
ensureFileBlobSchema,
migrationRows,
packFileJson,
referenceBlobs
} from '../db/file-blobs.js';
import { retentionClocks } from '../db/retention-clocks.js';
import { installHistoryCounts } from './history-counts.js';
import { migrationStep } from '../db/connection.js';
import type Database from 'better-sqlite3';
import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
import {
successfulObservationSql,
routineObservationSql,
operationTargetSql,
importantOperationSql
} from './observations.js';
const initialized = new WeakSet<Database.Database>();
export function ensureHostSchema(handle: Database.Database): void {
if (initialized.has(handle)) return;
// Background readers open only after controller migrations have completed.
if (handle.readonly) { initialized.add(handle); return; }
if (handle.readonly) {
initialized.add(handle);
return;
}
migrationStep('host schema and saved command files', () => migrateHostSchema(handle));
initialized.add(handle);
}
@@ -105,26 +118,39 @@ function migrateHostSchema(handle: Database.Database): void {
CREATE INDEX IF NOT EXISTS idx_ec_host_credentials_host ON ec_host_credentials(host_id);
CREATE INDEX IF NOT EXISTS idx_ec_host_enrollments_host ON ec_host_enrollments(host_id);
`);
const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
if (!columns.some(column => column.name === 'history_expired_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN history_expired_at TEXT');
if (!columns.some(column => column.name === 'request_hash')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN request_hash TEXT');
if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
if (!columns.some(column => column.name === 'observation_success')) {
const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as {
name: string;
}[];
if (!columns.some((column) => column.name === 'history_expired_at'))
handle.exec('ALTER TABLE ec_host_operations ADD COLUMN history_expired_at TEXT');
if (!columns.some((column) => column.name === 'request_hash'))
handle.exec('ALTER TABLE ec_host_operations ADD COLUMN request_hash TEXT');
if (!columns.some((column) => column.name === 'last_delivered_at'))
handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
if (!columns.some((column) => column.name === 'observation_success')) {
handle.transaction(() => {
handle.exec('ALTER TABLE ec_host_operations ADD COLUMN observation_success INTEGER NOT NULL DEFAULT 0');
handle.exec(`UPDATE ec_host_operations AS o SET observation_success=1 WHERE ${successfulObservationSql}`);
handle.exec(
'ALTER TABLE ec_host_operations ADD COLUMN observation_success INTEGER NOT NULL DEFAULT 0'
);
handle.exec(
`UPDATE ec_host_operations AS o SET observation_success=1 WHERE ${successfulObservationSql}`
);
})();
}
handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_observation_retention ON ec_host_operations(updated_at)
WHERE observation_success=1 AND state='completed'`);
if (!columns.some(column => column.name === 'routine')) {
if (!columns.some((column) => column.name === 'routine')) {
handle.transaction(() => {
handle.exec(`ALTER TABLE ec_host_operations ADD COLUMN routine INTEGER NOT NULL DEFAULT 0;
ALTER TABLE ec_host_operations ADD COLUMN server_name TEXT;
ALTER TABLE ec_host_operations ADD COLUMN target_path TEXT;
ALTER TABLE ec_host_operations ADD COLUMN source TEXT;`);
const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as { name: string }[];
const nativeInstances = ['id', 'server'].every(name => nativeColumns.some(column => column.name === name));
const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as {
name: string;
}[];
const nativeInstances = ['id', 'server'].every((name) =>
nativeColumns.some((column) => column.name === name)
);
handle.exec(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
target_path=${operationTargetSql}, server_name=${nativeInstances ? '(SELECT n.server FROM native_instances n WHERE n.id=o.instance_id)' : 'NULL'}`);
})();
@@ -135,59 +161,99 @@ function migrateHostSchema(handle: Database.Database): void {
CREATE INDEX IF NOT EXISTS ec_host_operations_important_history ON ec_host_operations(created_at)
WHERE ${importantOperationSql.replaceAll('o.', '')}`);
handle.exec('CREATE TABLE IF NOT EXISTS ec_host_schema_migrations (name TEXT PRIMARY KEY)');
if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('staged-operation-payloads')) {
if (
!handle
.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?')
.get('staged-operation-payloads')
) {
handle.transaction(() => {
handle.exec(`INSERT OR IGNORE INTO ec_host_operation_payloads(operation_id,payload)
SELECT id,payload FROM ec_host_operations WHERE kind='files.stage' AND json_type(payload,'$.files')='array';
UPDATE ec_host_operations SET payload=json_object('stageId',json_extract(payload,'$.stageId'))
WHERE kind='files.stage' AND json_type(payload,'$.files')='array';`);
handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('staged-operation-payloads');
handle
.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)')
.run('staged-operation-payloads');
})();
}
if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('completed-status-batches')) {
if (
!handle
.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?')
.get('completed-status-batches')
) {
handle.transaction(() => {
// Reclassify existing completed batches without discarding their per-server replies.
handle.exec(`UPDATE ec_host_operations AS o SET observation_success=1
WHERE o.kind='rcon.status-batch' AND o.observation_success=0 AND ${successfulObservationSql}`);
handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('completed-status-batches');
handle
.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)')
.run('completed-status-batches');
})();
}
const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[];
if (!allowanceColumns.some(column => column.name === 'usage_margin')) {
if (!hostColumns.some((column) => column.name === 'deleted_at'))
handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as {
name: string;
}[];
if (!allowanceColumns.some((column) => column.name === 'usage_margin')) {
// Existing caches include 25% padding. Preserve learned evidence while
// removing that padding exactly once, including dormant startup models.
handle.transaction(() => {
handle.exec('ALTER TABLE ec_resource_allowances ADD COLUMN usage_margin REAL NOT NULL DEFAULT 1');
handle.exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)');
handle.exec(
'ALTER TABLE ec_resource_allowances ADD COLUMN usage_margin REAL NOT NULL DEFAULT 1'
);
handle.exec(
'UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)'
);
})();
}
if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('operation-file-blobs')) {
if (
!handle
.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?')
.get('operation-file-blobs')
) {
handle.transaction(() => {
for (const row of migrationRows<{ id: string; payload: string; result: string | null }>(handle,
for (const row of migrationRows<{ id: string; payload: string; result: string | null }>(
handle,
`SELECT o.rowid AS rid,o.id,COALESCE(p.payload,o.payload) payload,o.result
FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id
WHERE o.rowid>? AND o.kind IN ('files.stage','instance.create','instance.rebuild','instance.restart','instance.start','files.read')
ORDER BY o.rowid LIMIT 1`)) {
ORDER BY o.rowid LIMIT 1`
)) {
const packed = packFileJson(JSON.parse(row.payload), handle);
handle.prepare('UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?) WHERE id=?')
handle
.prepare(
'UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?) WHERE id=?'
)
.run(createHash('sha256').update(row.payload).digest('hex'), row.id);
if (packed.hashes.size) {
handle.prepare(`INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)
ON CONFLICT(operation_id) DO UPDATE SET payload=excluded.payload`).run(row.id, JSON.stringify(packed.value));
handle.prepare("UPDATE ec_host_operations SET payload=json_remove(payload,'$.files') WHERE id=?").run(row.id);
handle
.prepare(
`INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)
ON CONFLICT(operation_id) DO UPDATE SET payload=excluded.payload`
)
.run(row.id, JSON.stringify(packed.value));
handle
.prepare(
"UPDATE ec_host_operations SET payload=json_remove(payload,'$.files') WHERE id=?"
)
.run(row.id);
referenceBlobs('operation', row.id, packed.hashes, handle);
}
if (row.result) {
const result = packFileJson(JSON.parse(row.result), handle);
if (result.hashes.size) {
handle.prepare('UPDATE ec_host_operations SET result=? WHERE id=?').run(JSON.stringify(result.value), row.id);
handle
.prepare('UPDATE ec_host_operations SET result=? WHERE id=?')
.run(JSON.stringify(result.value), row.id);
referenceBlobs('operation-result', row.id, result.hashes, handle);
}
}
}
handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('operation-file-blobs');
handle
.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)')
.run('operation-file-blobs');
})();
}
handle.exec(`CREATE TRIGGER IF NOT EXISTS release_operation_blobs AFTER DELETE ON ec_host_operations BEGIN
+164 -37
View File
@@ -6,25 +6,48 @@ const CACHE_MS = 5 * 60 * 1000;
const cache = new Map<string, { expires: number; release: Promise<GithubPluginRelease> }>();
export class GithubReleaseError extends Error {
constructor(message: string, public readonly status = 400) { super(message); }
constructor(
message: string,
public readonly status = 400
) {
super(message);
}
}
export function parseGithubReleaseUrl(input: string): { repository: string; tag: string | null } {
let url: URL;
try { url = new URL(input); } catch { throw new GithubReleaseError('Enter a GitHub repository or release URL.'); }
if (url.protocol !== 'https:' || url.hostname !== 'github.com' || url.port || url.username || url.password || url.search) {
try {
url = new URL(input);
} catch {
throw new GithubReleaseError('Enter a GitHub repository or release URL.');
}
if (
url.protocol !== 'https:' ||
url.hostname !== 'github.com' ||
url.port ||
url.username ||
url.password ||
url.search
) {
throw new GithubReleaseError('Use an https://github.com repository or release URL.');
}
const match = /^\/([\w-]+)\/([\w.-]+)(?:\/(.*?))?\/?$/.exec(url.pathname);
if (!match || match[2] === '.' || match[2] === '..') throw new GithubReleaseError('Invalid GitHub repository URL.');
if (!match || match[2] === '.' || match[2] === '..')
throw new GithubReleaseError('Invalid GitHub repository URL.');
const repository = `${match[1]}/${match[2]}`;
const path = (match[3] ?? '').replace(/\/$/, '');
let tag: string | null = null;
try {
if (path.startsWith('releases/tag/')) tag = decodeURIComponent(path.slice(13));
else if (path === 'releases' && url.hash.startsWith('#release-')) tag = decodeURIComponent(url.hash.slice(9));
else if (path === 'releases' && url.hash.startsWith('#release-'))
tag = decodeURIComponent(url.hash.slice(9));
else if (!['', 'releases', 'releases/latest'].includes(path) || url.hash) throw new Error();
} catch { throw new GithubReleaseError('Use a repository URL or a link to a specific GitHub release.'); }
} catch {
throw new GithubReleaseError(
'Use a repository URL or a link to a specific GitHub release.'
);
}
// eslint-disable-next-line no-control-regex -- Reject control characters in release metadata and asset names.
if (tag !== null && (!tag || tag.length > 80 || /[\u0000-\u001f\u007f]/.test(tag))) {
throw new GithubReleaseError('Release tags must contain 1–80 characters on one line.');
}
@@ -32,7 +55,12 @@ export function parseGithubReleaseUrl(input: string): { repository: string; tag:
}
function safeAssetName(name: unknown): name is string {
return typeof name === 'string' && /^[^<>:"/\\|?*\u0000-\u001f]+\.dll$/i.test(name) && name.length <= 200;
return (
typeof name === 'string' &&
// eslint-disable-next-line no-control-regex -- Reject control characters in release metadata and asset names.
/^[^<>:"/\\|?*\u0000-\u001f]+\.dll$/i.test(name) &&
name.length <= 200
);
}
async function limitedBody(response: Response, limit: number): Promise<Buffer> {
@@ -49,45 +77,102 @@ async function limitedBody(response: Response, limit: number): Promise<Buffer> {
const next = await reader.read();
if (next.done) break;
size += next.value.byteLength;
if (size > limit) throw new GithubReleaseError('GitHub response exceeds the download size limit.', 502);
if (size > limit)
throw new GithubReleaseError(
'GitHub response exceeds the download size limit.',
502
);
chunks.push(next.value);
}
} finally { await reader.cancel(); }
} finally {
await reader.cancel();
}
return Buffer.concat(chunks);
}
async function fetchRelease(repository: string, tag: string | null): Promise<GithubPluginRelease> {
const suffix = tag === null ? 'latest' : `tags/${encodeURIComponent(tag)}`;
try {
const response = await fetch(`https://api.github.com/repos/${repository}/releases/${suffix}`, {
headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'Electric-Cherry', 'X-GitHub-Api-Version': '2022-11-28' },
signal: AbortSignal.timeout(15_000), redirect: 'error'
});
const response = await fetch(
`https://api.github.com/repos/${repository}/releases/${suffix}`,
{
headers: {
Accept: 'application/vnd.github+json',
'User-Agent': 'Electric-Cherry',
'X-GitHub-Api-Version': '2022-11-28'
},
signal: AbortSignal.timeout(15_000),
redirect: 'error'
}
);
if (!response.ok) {
await response.body?.cancel();
if (response.status === 404) throw new GithubReleaseError('No published release found. Check the repository and tag; only public repositories are supported.', 404);
if ([403, 429].includes(response.status)) throw new GithubReleaseError('GitHub is limiting requests. Try checking again later.', 429);
throw new GithubReleaseError(`GitHub could not load this release (HTTP ${response.status}).`, 502);
if (response.status === 404)
throw new GithubReleaseError(
'No published release found. Check the repository and tag; only public repositories are supported.',
404
);
if ([403, 429].includes(response.status))
throw new GithubReleaseError(
'GitHub is limiting requests. Try checking again later.',
429
);
throw new GithubReleaseError(
`GitHub could not load this release (HTTP ${response.status}).`,
502
);
}
const data = JSON.parse((await limitedBody(response, 2 * 1024 * 1024)).toString('utf8'));
if (!Number.isSafeInteger(data.id) || data.id < 1 || data.draft || typeof data.tag_name !== 'string' ||
!data.tag_name.trim() || data.tag_name.length > 80 || /[\u0000-\u001f\u007f]/.test(data.tag_name) ||
!Number.isFinite(Date.parse(data.published_at)) || !Array.isArray(data.assets)) {
if (
!Number.isSafeInteger(data.id) ||
data.id < 1 ||
data.draft ||
typeof data.tag_name !== 'string' ||
!data.tag_name.trim() ||
data.tag_name.length > 80 ||
// eslint-disable-next-line no-control-regex -- Reject control characters in release metadata and asset names.
/[\u0000-\u001f\u007f]/.test(data.tag_name) ||
!Number.isFinite(Date.parse(data.published_at)) ||
!Array.isArray(data.assets)
) {
throw new GithubReleaseError('GitHub returned invalid release metadata.', 502);
}
return {
repository, id: data.id, tag: data.tag_name,
repository,
id: data.id,
tag: data.tag_name,
url: `https://github.com/${repository}/releases/tag/${encodeURIComponent(data.tag_name)}`,
name: typeof data.name === 'string' ? data.name : data.tag_name,
notes: typeof data.body === 'string' ? data.body : '', publishedAt: data.published_at,
notes: typeof data.body === 'string' ? data.body : '',
publishedAt: data.published_at,
prerelease: data.prerelease === true,
assets: data.assets.filter((a: any) => a && Number.isSafeInteger(a.id) && a.id > 0 && safeAssetName(a.name) &&
a.state === 'uploaded' && Number.isSafeInteger(a.size) && a.size > 0 && a.size <= MAX_DLL_BYTES)
.map((a: any) => ({ id: a.id, name: a.name, size: a.size, digest: typeof a.digest === 'string' ? a.digest : null }))
assets: data.assets
.filter(
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Validate dynamically decoded GitHub asset fields below.
(a: any) =>
a &&
Number.isSafeInteger(a.id) &&
a.id > 0 &&
safeAssetName(a.name) &&
a.state === 'uploaded' &&
Number.isSafeInteger(a.size) &&
a.size > 0 &&
a.size <= MAX_DLL_BYTES
)
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- The preceding filter validates the decoded asset fields.
.map((a: any) => ({
id: a.id,
name: a.name,
size: a.size,
digest: typeof a.digest === 'string' ? a.digest : null
}))
};
} catch (error) {
if (error instanceof GithubReleaseError) throw error;
throw new GithubReleaseError('Could not reach GitHub or read the release. Try again shortly.', 502);
throw new GithubReleaseError(
'Could not reach GitHub or read the release. Try again shortly.',
502
);
}
}
@@ -99,20 +184,46 @@ export async function getGithubRelease(url: string, fresh = false): Promise<Gith
if (cache.size >= 100) cache.delete(cache.keys().next().value!);
const release = fetchRelease(repository, tag);
cache.set(key, { expires: Date.now() + CACHE_MS, release });
try { return await release; } catch (error) { cache.delete(key); throw error; }
try {
return await release;
} catch (error) {
cache.delete(key);
throw error;
}
}
export async function downloadGithubAsset(release: GithubPluginRelease, asset: GithubPluginAsset): Promise<Buffer> {
export async function downloadGithubAsset(
release: GithubPluginRelease,
asset: GithubPluginAsset
): Promise<Buffer> {
// Construct the URL from validated metadata. Never fetch a client-supplied download URL.
let url = new URL(`https://github.com/${release.repository}/releases/download/${encodeURIComponent(release.tag)}/${encodeURIComponent(asset.name)}`);
let url = new URL(
`https://github.com/${release.repository}/releases/download/${encodeURIComponent(release.tag)}/${encodeURIComponent(asset.name)}`
);
const signal = AbortSignal.timeout(60_000);
try {
for (let i = 0; i < 5; i++) {
if (url.protocol !== 'https:' || url.port || url.username || url.password ||
!['github.com', 'release-assets.githubusercontent.com', 'objects.githubusercontent.com'].includes(url.hostname)) {
throw new GithubReleaseError('GitHub redirected this download to an unsupported host.', 502);
if (
url.protocol !== 'https:' ||
url.port ||
url.username ||
url.password ||
![
'github.com',
'release-assets.githubusercontent.com',
'objects.githubusercontent.com'
].includes(url.hostname)
) {
throw new GithubReleaseError(
'GitHub redirected this download to an unsupported host.',
502
);
}
const response = await fetch(url, { redirect: 'manual', signal, headers: { 'User-Agent': 'Electric-Cherry' } });
const response = await fetch(url, {
redirect: 'manual',
signal,
headers: { 'User-Agent': 'Electric-Cherry' }
});
if ([301, 302, 303, 307, 308].includes(response.status)) {
await response.body?.cancel();
const location = response.headers.get('location');
@@ -122,19 +233,35 @@ export async function downloadGithubAsset(release: GithubPluginRelease, asset: G
}
if (!response.ok) {
await response.body?.cancel();
throw new GithubReleaseError(`GitHub asset download failed (HTTP ${response.status}).`, 502);
throw new GithubReleaseError(
`GitHub asset download failed (HTTP ${response.status}).`,
502
);
}
const content = await limitedBody(response, MAX_DLL_BYTES);
if (content.length !== asset.size) throw new GithubReleaseError('Downloaded size differs from the release asset. Preview the release again.', 502);
if (content.length !== asset.size)
throw new GithubReleaseError(
'Downloaded size differs from the release asset. Preview the release again.',
502
);
const sha = createHash('sha256').update(content).digest('hex');
if (asset.digest?.startsWith('sha256:') && asset.digest.toLowerCase() !== `sha256:${sha}`) {
throw new GithubReleaseError('Downloaded checksum does not match GitHub’s asset digest.', 502);
if (
asset.digest?.startsWith('sha256:') &&
asset.digest.toLowerCase() !== `sha256:${sha}`
) {
throw new GithubReleaseError(
'Downloaded checksum does not match GitHub’s asset digest.',
502
);
}
return content;
}
throw new GithubReleaseError('GitHub asset download redirected too many times.', 502);
} catch (error) {
if (error instanceof GithubReleaseError) throw error;
throw new GithubReleaseError('Could not download the DLL from GitHub. Try again shortly.', 502);
throw new GithubReleaseError(
'Could not download the DLL from GitHub. Try again shortly.',
502
);
}
}
+120 -33
View File
@@ -22,53 +22,119 @@ export function freshRelease(): number | null {
}
export function runningVersions(serverName?: string): PlutoniumVersion[] {
const bindings = serverName === undefined ? [] : [serverName];
const versions = db().prepare(`SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}`)
const versions = db()
.prepare(
`SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}`
)
.all(...bindings) as PlutoniumVersion[];
const target = freshRelease();
const byServer = new Map(versions.map(version => [version.server, version]));
const epochs = serverName === undefined ? latestGameSubmissions() : new Map([[serverName, latestGameSubmission(serverName)]]);
const recovering = new Set((db().prepare(`SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
.all(...bindings) as { server: string }[]).map(row => row.server));
const byServer = new Map(versions.map((version) => [version.server, version]));
const epochs =
serverName === undefined
? latestGameSubmissions()
: new Map([[serverName, latestGameSubmission(serverName)]]);
const recovering = new Set(
(
db()
.prepare(
`SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`
)
.all(...bindings) as { server: string }[]
).map((row) => row.server)
);
const selected = serverName === undefined ? undefined : getServer(serverName);
const servers = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
return servers.map(server => {
const value = byServer.get(server.name)
?? { server: server.name, revision: null, checked_at: null, error: null };
const servers =
serverName === undefined
? listServers()
: selected && !selected.destroyed_at
? [selected]
: [];
return servers.map((server) => {
const value = byServer.get(server.name) ?? {
server: server.name,
revision: null,
checked_at: null,
error: null
};
const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN;
const epoch = epochs.get(server.name);
const stopped = server.desired_state === 'stopped';
const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !recovering.has(server.name);
const valid =
!value.error &&
age >= 0 &&
age < HOUR &&
(!epoch || value.checked_at! > epoch) &&
!recovering.has(server.name);
const revision = valid && !stopped ? value.revision : null;
return { ...value, revision, stopped,
update_status: stopped ? 'stopped' : revision === null || target === null ? 'unknown'
: revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' };
return {
...value,
revision,
stopped,
update_status: stopped
? 'stopped'
: revision === null || target === null
? 'unknown'
: revision < target
? 'outdated'
: revision > target
? 'ahead'
: 'current'
};
});
}
export async function probeVersion(server: string, options: { maintenance?: boolean; operationId?: string } = {}): Promise<number | null> {
export async function probeVersion(
server: string,
options: { maintenance?: boolean; operationId?: string } = {}
): Promise<number | null> {
// Startup verification calls this directly; only maintenance observers yield
// ownership, without replacing provisioning's version evidence with an error.
const suppressed = () => (options.maintenance && hasActiveProvisioning(server)) || (options.operationId !== undefined
&& !db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL`).get(options.operationId, server));
const suppressed = () =>
(options.maintenance && hasActiveProvisioning(server)) ||
(options.operationId !== undefined &&
!db()
.prepare(
`SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL`
)
.get(options.operationId, server));
// The verifying operation keeps its recovery slot until every readiness
// check passes. Waiting on its own slot would deadlock native rebuilds.
const recoveryBlocked = () => restartRecoveryPending(server) && !(options.operationId
&& db().prepare('SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL').get(server, options.operationId));
const recoveryBlocked = () =>
restartRecoveryPending(server) &&
!(
options.operationId &&
db()
.prepare(
'SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL'
)
.get(server, options.operationId)
);
if (suppressed()) return null;
const original = getServer(server);
if (!original) return null;
const epoch = latestGameSubmission(server);
const recovering = recoveryBlocked();
const health = !recovering && original.desired_state !== 'stopped'
? await nativeHealth(server).catch(() => ({ ok: false })) : { ok: false };
const health =
!recovering && original.desired_state !== 'stopped'
? await nativeHealth(server).catch(() => ({ ok: false }))
: { ok: false };
if (suppressed()) return null;
let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' }
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
let result = !health.ok
? { revision: null, error: 'Waiting for a healthy running server.' }
: await rconPlutoniumVersion(server).catch(() => ({
revision: null,
error: 'Version check failed.'
}));
if (suppressed()) return null;
const current = getServer(server);
// An older probe must not clear evidence already recorded for a new process
// or replacement registration, including replacements with no submission yet.
if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null;
if (
!current ||
current.created_at !== original.created_at ||
epoch !== latestGameSubmission(server)
)
return null;
const checkedAt = nowIso();
if (recoveryBlocked()) {
result = { revision: null, error: 'Waiting for restart recovery.' };
@@ -78,8 +144,11 @@ export async function probeVersion(server: string, options: { maintenance?: bool
result = { revision: null, error: 'Waiting for a version observation after startup.' };
}
// Clear the old revision on failure: it is no longer evidence of what is running.
db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?)
ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`)
db()
.prepare(
`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?)
ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`
)
.run(server, result.revision, checkedAt, result.error);
return result.revision;
}
@@ -87,14 +156,18 @@ export async function probeVersion(server: string, options: { maintenance?: bool
let checking: Promise<void> | null = null;
export function checkPlutonium(): Promise<void> {
if (checking) return checking;
checking = check().finally(() => { checking = null; });
checking = check().finally(() => {
checking = null;
});
return checking;
}
async function check(): Promise<void> {
await checkRelease();
const servers = listServers();
for (let i = 0; i < servers.length; i += 4) {
await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name, { maintenance: true })));
await Promise.all(
servers.slice(i, i + 4).map((s) => probeVersion(s.name, { maintenance: true }))
);
}
writeSetting('plutonium_versions_checked_at', nowIso());
}
@@ -103,19 +176,33 @@ let checkingRelease: Promise<void> | null = null;
/** Refresh the published target without probing unrelated fleet servers. */
export function checkRelease(): Promise<void> {
if (checkingRelease) return checkingRelease;
checkingRelease = fetchRelease().finally(() => { checkingRelease = null; });
checkingRelease = fetchRelease().finally(() => {
checkingRelease = null;
});
return checkingRelease;
}
async function fetchRelease(): Promise<void> {
const previous = release();
try {
const response = await fetch(PLUTONIUM_MANIFEST_URL, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' });
if (!response.ok) throw new Error(`Updater manifest request failed (HTTP ${response.status}).`);
const response = await fetch(PLUTONIUM_MANIFEST_URL, {
signal: AbortSignal.timeout(config.httpTimeoutMs),
redirect: 'error'
});
if (!response.ok)
throw new Error(`Updater manifest request failed (HTTP ${response.status}).`);
const body = await response.text();
if (body.length > 2_000_000) throw new Error('Updater manifest response was too large.');
writeSetting('plutonium_release', { revision: parsePlutoniumManifest(JSON.parse(body)), checkedAt: nowIso(), error: null });
writeSetting('plutonium_release', {
revision: parsePlutoniumManifest(JSON.parse(body)),
checkedAt: nowIso(),
error: null
});
} catch (err) {
writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message });
writeSetting('plutonium_release', {
...previous,
checkedAt: nowIso(),
error: (err as Error).message
});
}
}
@@ -2,10 +2,15 @@ import type { RconStatusResult } from './rcon.js';
/** Redact before truncating or hex encoding, including IPv6 and mapped IPv4 endpoints. */
export function redactRconAddresses(text: string): string {
return text.replace(/\[[0-9a-f:.]+(?:%[\w.-]+)?\](?::\d+)?|(?<![\w:])(?:[0-9a-f]{0,4}:){2,}[0-9a-f:.]*(?:%[\w.-]+)?|\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b/gi, '[redacted]');
return text.replace(
/\[[0-9a-f:.]+(?:%[\w.-]+)?\](?::\d+)?|(?<![\w:])(?:[0-9a-f]{0,4}:){2,}[0-9a-f:.]*(?:%[\w.-]+)?|\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b/gi,
'[redacted]'
);
}
/** Successful checks clear diagnostics; only sanitized failure evidence is retained. */
export function failureReplySnippet(result: RconStatusResult): string | null {
return !result.ok && result.rawReply ? redactRconAddresses(result.rawReply).slice(0, 4000) : null;
return !result.ok && result.rawReply
? redactRconAddresses(result.rawReply).slice(0, 4000)
: null;
}
+155 -41
View File
@@ -21,7 +21,8 @@ const PREFIX = Buffer.from([0xff, 0xff, 0xff, 0xff]);
// Distinct RCON failure modes. Surfaced to the UI so the operator can tell
// "wrong password" from "wrong host" from "Plutonium isn't listening".
export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
export type RconFailure =
'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
export interface RconEndpoint {
host: string;
@@ -57,7 +58,12 @@ async function sendRcon(host: string, port: number, command: string): Promise<Se
};
const timer = setTimeout(
() => finish({ ok: false, reason: 'timeout', detail: `no reply within ${config.rcon.timeoutMs}ms` }),
() =>
finish({
ok: false,
reason: 'timeout',
detail: `no reply within ${config.rcon.timeoutMs}ms`
}),
config.rcon.timeoutMs
);
@@ -159,8 +165,8 @@ export function parseStatusHeader(text: string): Record<string, string> {
const out: Record<string, string> = {};
for (const rawLine of text.split(/\r?\n/)) {
const line = rawLine.trim();
if (/^-+\s+-+/.test(line)) break; // scoreboard row — stop
if (/^\d+\s+/.test(line)) break; // player row that snuck in
if (/^-+\s+-+/.test(line)) break; // scoreboard row — stop
if (/^\d+\s+/.test(line)) break; // player row that snuck in
const m = /^([a-z][a-z0-9_]*)\s*:\s*(.+)$/i.exec(line);
if (m) out[m[1].toLowerCase()] = m[2].trim();
}
@@ -168,10 +174,10 @@ export function parseStatusHeader(text: string): Record<string, string> {
}
export interface RconPlayer {
num: number | null; // client slot — print dialect only
num: number | null; // client slot — print dialect only
score: number;
ping: number;
guid: string | null; // format varies by engine (hex, decimal, steam ID)
guid: string | null; // format varies by engine (hex, decimal, steam ID)
name: string;
}
@@ -186,7 +192,7 @@ export function stripColorCodes(s: string): string {
interface Column {
name: string;
start: number;
end: number; // exclusive
end: number; // exclusive
}
/**
@@ -261,7 +267,8 @@ export function parsePlayerRoster(text: string): RconPlayer[] {
const num = Number(numStr);
const score = Number(slice(line, scoreI));
const ping = Number(slice(line, pingI));
if (!Number.isFinite(num) || !Number.isFinite(score) || !Number.isFinite(ping)) continue;
if (!Number.isFinite(num) || !Number.isFinite(score) || !Number.isFinite(ping))
continue;
const guid = guidI >= 0 ? slice(line, guidI) || null : null;
const name = stripColorCodes(slice(line, nameI));
@@ -325,10 +332,14 @@ const statusInFlight = new Map<string, Promise<RconStatusResult>>();
async function sendStatus(endpoint: RconEndpoint): Promise<SendResult> {
for (let attempt = 0; ; attempt++) {
const result = await sendRcon(endpoint.host, endpoint.port, 'status');
if (result.ok || (result.reason !== 'timeout' && result.reason !== 'send-error') || attempt === 2) {
if (
result.ok ||
(result.reason !== 'timeout' && result.reason !== 'send-error') ||
attempt === 2
) {
return result;
}
await new Promise<void>(resolve => setTimeout(resolve, 250 * 2 ** attempt));
await new Promise<void>((resolve) => setTimeout(resolve, 250 * 2 ** attempt));
}
}
@@ -343,8 +354,12 @@ export function formatRawReply(buf: Buffer): string {
const lines: string[] = [];
for (let i = 0; i < bytes.length; i += 16) {
const chunk = bytes.subarray(i, i + 16);
const hex = Array.from(chunk).map((b) => b.toString(16).padStart(2, '0')).join(' ');
const ascii = Array.from(chunk).map((b) => (b >= 0x20 && b < 0x7f ? String.fromCharCode(b) : '.')).join('');
const hex = Array.from(chunk)
.map((b) => b.toString(16).padStart(2, '0'))
.join(' ');
const ascii = Array.from(chunk)
.map((b) => (b >= 0x20 && b < 0x7f ? String.fromCharCode(b) : '.'))
.join('');
lines.push(`${i.toString(16).padStart(4, '0')} ${hex.padEnd(47, ' ')} ${ascii}`);
}
if (buf.length > RAW_REPLY_LIMIT) {
@@ -353,28 +368,70 @@ export function formatRawReply(buf: Buffer): string {
return lines.join('\n');
}
export interface AgentRconReply { ok?: boolean; reason?: string; text: string }
export interface AgentRconReply {
ok?: boolean;
reason?: string;
text: string;
}
export function agentRconStatusResult(reply: AgentRconReply, endpoint: { host: string; port: number }): RconStatusResult {
export function agentRconStatusResult(
reply: AgentRconReply,
endpoint: { host: string; port: number }
): RconStatusResult {
if (reply.ok === false) {
const failures: Record<string, { reason: RconFailure; detail: string }> = {
unconfigured: { reason: 'no-config', detail: 'Agent RCON password is not configured.' },
unreachable: { reason: 'send-error', detail: 'Agent could not reach the local RCON endpoint.' },
timeout: { reason: 'timeout', detail: 'Agent received no reply from the local RCON endpoint.' },
unreachable: {
reason: 'send-error',
detail: 'Agent could not reach the local RCON endpoint.'
},
timeout: {
reason: 'timeout',
detail: 'Agent received no reply from the local RCON endpoint.'
},
authentication: { reason: 'bad-reply', detail: 'wrong RCON password' },
'invalid-reply': { reason: 'bad-reply', detail: 'Agent RCON reply is missing the 0xff prefix.' }
'invalid-reply': {
reason: 'bad-reply',
detail: 'Agent RCON reply is missing the 0xff prefix.'
}
};
return {
ok: false,
endpoint,
...(failures[reply.reason ?? ''] ?? {
reason: 'bad-reply' as const,
detail: 'Agent RCON query failed.'
})
};
return { ok: false, endpoint, ...(failures[reply.reason ?? ''] ?? {
reason: 'bad-reply' as const, detail: 'Agent RCON query failed.'
}) };
}
const players = parsePlayerCount(reply.text);
if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: redactRconAddresses(reply.text).slice(0, 4000) };
const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text);
return { ok: true, endpoint, status: { players,
capacity: positiveCapacity(info.sv_maxclients ?? header.sv_maxclients),
map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null,
gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null, roster: parsePlayerRoster(reply.text) } };
if (players === null)
return {
ok: false,
endpoint,
reason: 'bad-reply',
detail: 'Agent returned an unrecognized RCON status.',
rawReply: redactRconAddresses(reply.text).slice(0, 4000)
};
const info = parseInfoLine(reply.text);
const header = parseStatusHeader(reply.text);
return {
ok: true,
endpoint,
status: {
players,
capacity: positiveCapacity(info.sv_maxclients ?? header.sv_maxclients),
map:
info.mapname ??
info.sv_mapname ??
header.map ??
header.mapname ??
header.sv_mapname ??
null,
gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null,
roster: parsePlayerRoster(reply.text)
}
};
}
/**
@@ -386,17 +443,29 @@ export function agentRconStatusResult(reply: AgentRconReply, endpoint: { host: s
export async function rconStatusResult(server: string): Promise<RconStatusResult> {
const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null;
if (native) {
const key = JSON.stringify(['agent', native.id, native.generation, latestGameSubmission(server)]);
const key = JSON.stringify([
'agent',
native.id,
native.generation,
latestGameSubmission(server)
]);
const existing = statusInFlight.get(key);
if (existing) return existing;
const endpoint = { host: `agent:${native.host_id}`, port: native.port };
const request = (async (): Promise<RconStatusResult> => {
try {
const reply = await nativeRequest<{ ok?: boolean; reason?: string; text: string }>(native, 'rcon.execute', { command: 'status' });
const reply = await nativeRequest<{ ok?: boolean; reason?: string; text: string }>(
native,
'rcon.execute',
{ command: 'status' }
);
return agentRconStatusResult(reply, endpoint);
} catch (error) { return { ok: false, endpoint, reason: 'timeout', detail: (error as Error).message }; }
} catch (error) {
return { ok: false, endpoint, reason: 'timeout', detail: (error as Error).message };
}
})().finally(() => statusInFlight.delete(key));
statusInFlight.set(key, request); return request;
statusInFlight.set(key, request);
return request;
}
// Snapshot before any network work. A probe begun after a restart or a
// replacement registration must not join one observing the old process.
@@ -405,16 +474,28 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
const res = await endpointFor(server);
if (!res.endpoint) {
if (res.reason === 'unknown-server') {
return { ok: false, endpoint: null, reason: 'no-port', detail: `unknown server: ${server}` };
return {
ok: false,
endpoint: null,
reason: 'no-port',
detail: `unknown server: ${server}`
};
}
return { ok: false, endpoint: null, reason: 'no-port', detail: 'No game port is registered for this server.' };
return {
ok: false,
endpoint: null,
reason: 'no-port',
detail: 'No game port is registered for this server.'
};
}
const ep = res.endpoint;
const key = JSON.stringify([server, createdAt, epoch, ep.host, ep.port]);
const pending = statusInFlight.get(key);
if (pending) return pending;
const request = probeStatus(ep).finally(() => { statusInFlight.delete(key); });
const request = probeStatus(ep).finally(() => {
statusInFlight.delete(key);
});
statusInFlight.set(key, request);
return request;
}
@@ -459,7 +540,13 @@ async function probeStatus(ep: RconEndpoint): Promise<RconStatusResult> {
} else {
detail = `unparseable reply (${send.buf.length} bytes) — see snippet below`;
}
return { ok: false, endpoint: ep, reason: 'bad-reply', detail, rawReply: formatRawReply(send.buf) };
return {
ok: false,
endpoint: ep,
reason: 'bad-reply',
detail,
rawReply: formatRawReply(send.buf)
};
}
const header = parseStatusHeader(text);
return {
@@ -469,8 +556,19 @@ async function probeStatus(ep: RconEndpoint): Promise<RconStatusResult> {
// Prefer statusResponse `\info\` line values when present, fall
// back to print-body `key: value` header lines otherwise.
players,
capacity: positiveCapacity(info.sv_maxclients ?? info.sv_maxClients ?? header.sv_maxclients ?? header.sv_maxClients),
map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null,
capacity: positiveCapacity(
info.sv_maxclients ??
info.sv_maxClients ??
header.sv_maxclients ??
header.sv_maxClients
),
map:
info.mapname ??
info.sv_mapname ??
header.map ??
header.mapname ??
header.sv_mapname ??
null,
gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null,
roster: parsePlayerRoster(text)
}
@@ -483,14 +581,26 @@ function positiveCapacity(value: string | undefined): number | null {
}
/** Query the running process, not the updater's on-disk files. */
export async function rconPlutoniumVersion(server: string): Promise<{ revision: number | null; error: string | null }> {
export async function rconPlutoniumVersion(
server: string
): Promise<{ revision: number | null; error: string | null }> {
const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null;
if (native) {
try {
const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'version' });
const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', {
command: 'version'
});
const revision = parsePlutoniumRevision(reply.text);
return { revision, error: revision === null ? 'Agent RCON reply did not identify a runtime revision.' : null };
} catch (error) { return { revision: null, error: (error as Error).message }; }
return {
revision,
error:
revision === null
? 'Agent RCON reply did not identify a runtime revision.'
: null
};
} catch (error) {
return { revision: null, error: (error as Error).message };
}
}
const { endpoint } = await endpointFor(server);
if (!endpoint) return { revision: null, error: 'No RCON endpoint available.' };
@@ -498,5 +608,9 @@ export async function rconPlutoniumVersion(server: string): Promise<{ revision:
if (!response.ok) return { revision: null, error: response.detail };
const body = stripPrefix(response.buf);
const revision = body?.trimStart().startsWith('print') ? parsePlutoniumRevision(body) : null;
return { revision, error: revision === null ? 'RCON version reply did not identify a Plutonium revision.' : null };
return {
revision,
error:
revision === null ? 'RCON version reply did not identify a Plutonium revision.' : null
};
}
+7 -2
View File
@@ -4,10 +4,15 @@ import { maintenanceTick, plutoniumStatus } from './runtime.js';
import type { MaintenanceOverview } from '../../maintenance.js';
export function maintenanceOverview(selectedJob: number | null = null): MaintenanceOverview {
return { policies: maintenancePolicies(), recovery: watchdogOverview(), runtime: plutoniumStatus(selectedJob) };
return {
policies: maintenancePolicies(),
recovery: watchdogOverview(),
runtime: plutoniumStatus(selectedJob)
};
}
export async function runMaintenance(): Promise<void> {
// One failing observer must not prevent the other capabilities from reconciling.
const results = await Promise.allSettled([maintenanceTick(), watchdogTick()]);
for (const result of results) if (result.status === 'rejected') console.error('[maintenance]', result.reason);
for (const result of results)
if (result.status === 'rejected') console.error('[maintenance]', result.reason);
}

Some files were not shown because too many files have changed in this diff Show More