Add native host management and role-based operator authentication
Introduce an outbound Python host agent, Debian installer, durable remote operations, immutable game-file libraries, host-aware deployments and ports, and occupancy-safe campaigns and draining. Native hosts do not require Komodo. Add administrator, operator and viewer accounts, protected management routes, secure sessions, password recovery, audit history and separate telemetry tokens. Document first-admin setup, reporter migration and live-host acceptance checks. Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type checking, the production build, compiled SSR/API smoke tests and administrator recovery. Real Debian/Docker T5/T6 acceptance remains unrun.
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
import type { UserRole } from '../../auth.js';
|
||||
|
||||
export type RoutePermission = 'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin';
|
||||
|
||||
const within = (route: string, base: string): boolean => route === base || route.startsWith(base + '/');
|
||||
const readMethod = (method: string): boolean => method === 'GET' || method === 'HEAD';
|
||||
|
||||
/** Classify SvelteKit's resolved route ID, never a user-supplied URL pathname.
|
||||
* Route IDs preserve the same policy for page-data requests and encoded URLs.
|
||||
* Machine endpoints enforce their own host/enrollment/telemetry credentials.
|
||||
*/
|
||||
export function routePermission(routeId: string | null, method: string): RoutePermission {
|
||||
method = method.toUpperCase();
|
||||
const reading = readMethod(method);
|
||||
if (routeId === null) return reading ? 'viewer' : 'operator';
|
||||
// These cannot occur in this application's resolved route IDs. Fail closed
|
||||
// if a future caller accidentally supplies a raw or noncanonical pathname.
|
||||
if (!routeId.startsWith('/') || /[%\\?#\x00-\x1f\x7f]/.test(routeId)
|
||||
|| routeId !== '/' && routeId.split('/').slice(1).some(part => !part || part === '.' || part === '..')) return 'admin';
|
||||
|
||||
if (within(routeId, '/api/agent')) return 'machine';
|
||||
if (routeId === '/api/telemetry' && method === 'POST') return 'machine';
|
||||
if ((routeId === '/login' || routeId === '/setup') && reading) return 'public';
|
||||
if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST') return 'public';
|
||||
if (routeId === '/account' || ['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId)) return 'authenticated';
|
||||
|
||||
if (within(routeId, '/api/users') || within(routeId, '/settings') || within(routeId, '/api/auth')
|
||||
|| within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin';
|
||||
|
||||
if (!reading && ['/api/hosts', '/api/settings', '/api/game-files'].some(base => within(routeId, base))) return 'admin';
|
||||
// These endpoints mix operational actions with global policy and schedule
|
||||
// changes. Per-incident/job actions remain available to operators.
|
||||
if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) return 'admin';
|
||||
|
||||
// Summaries remain visible to viewers; raw configuration, historical file
|
||||
// bytes, and unfiltered diagnostics may contain runtime credentials.
|
||||
if (routeId === '/configs/[id]' || routeId === '/api/configs/[id]' || routeId === '/api/configs/inspect'
|
||||
|| routeId === '/api/campaigns/[id]/file/[fileId]' || routeId === '/api/deployments/[id]/file/[fileId]'
|
||||
|| routeId === '/api/servers/[name]/logs/[file]' || routeId === '/api/servers/[name]/telemetry-requests') return 'operator';
|
||||
|
||||
return reading ? 'viewer' : 'operator';
|
||||
}
|
||||
|
||||
/** A browser role never substitutes for a machine credential. */
|
||||
export function roleAllows(role: UserRole, permission: RoutePermission): boolean {
|
||||
if (!['viewer', 'operator', 'admin'].includes(role)) return false;
|
||||
if (permission === 'machine') return false;
|
||||
if (permission === 'public' || permission === 'authenticated' || permission === 'viewer') return true;
|
||||
if (permission === 'operator') return role === 'operator' || role === 'admin';
|
||||
return permission === 'admin' && role === 'admin';
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import { error, json, redirect, type Handle } from '@sveltejs/kit';
|
||||
import { readSession, readSessionHash, recordAudit, setupRequired } from './index.js';
|
||||
import { authBody, authError, requireSameOrigin, requireUserTransport, sessionCookie } from './http.js';
|
||||
import { roleAllows, routePermission, type RoutePermission } from './access.js';
|
||||
|
||||
function publicAsset(routeId: string | null, path: string, method: string) {
|
||||
return routeId === null && ['GET','HEAD'].includes(method) &&
|
||||
(path.startsWith('/_app/') || path === '/favicon.svg' || ['/agent/ec-agent.py','/agent/install.sh','/agent/manifest.json'].includes(path));
|
||||
}
|
||||
|
||||
/** Streaming responses must lose access when the underlying session does. */
|
||||
export function protectEventStream(response: Response, tokenHash: string, permission: RoutePermission, signal: AbortSignal): Response {
|
||||
if (!response.body) return response;
|
||||
const reader = response.body.getReader();
|
||||
let timer: ReturnType<typeof setInterval> | undefined;
|
||||
let closed = false;
|
||||
let closeStream: () => void = () => {};
|
||||
const cleanup = () => { if (timer) clearInterval(timer); signal.removeEventListener('abort', closeStream); };
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
closeStream = () => {
|
||||
if (closed) return; closed = true; cleanup();
|
||||
void reader.cancel().catch(() => {}); controller.close();
|
||||
};
|
||||
timer = setInterval(() => {
|
||||
try {
|
||||
const current = readSessionHash(tokenHash);
|
||||
if (!current || current.user.mustChangePassword || !roleAllows(current.user.role,permission)) closeStream();
|
||||
} catch { closeStream(); }
|
||||
},15_000);
|
||||
signal.addEventListener('abort',closeStream,{once:true});
|
||||
if (signal.aborted) { closeStream(); return; }
|
||||
},
|
||||
async pull(controller) {
|
||||
try {
|
||||
const next = await reader.read();
|
||||
if (closed) return;
|
||||
if (next.done) { closed=true; cleanup(); controller.close(); }
|
||||
else controller.enqueue(next.value);
|
||||
} catch (cause) { if (!closed) { closed=true; cleanup(); controller.error(cause); } }
|
||||
},
|
||||
cancel() { closed=true; cleanup(); return reader.cancel(); }
|
||||
});
|
||||
return new Response(stream,{status:response.status,statusText:response.statusText,headers:response.headers});
|
||||
}
|
||||
|
||||
export const authenticationHandle: Handle = async ({ event, resolve }) => {
|
||||
event.locals.user = null; event.locals.sessionHash = null;
|
||||
const routeId = event.route.id;
|
||||
const method = event.request.method.toUpperCase();
|
||||
if (publicAsset(routeId,event.url.pathname,method)) return resolve(event);
|
||||
const permission = routePermission(routeId,method);
|
||||
if (!routeId?.startsWith('/api/agent/') && /^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization')??'')) {
|
||||
return json({error:'Host credentials only authorize the host agent API.'},{status:403});
|
||||
}
|
||||
if (permission === 'machine') return resolve(event);
|
||||
const api = routeId?.startsWith('/api/') || event.url.pathname.startsWith('/api/');
|
||||
try {
|
||||
requireUserTransport(event.url);
|
||||
if (!['GET','HEAD','OPTIONS'].includes(method)) requireSameOrigin(event.request,event.url);
|
||||
} catch (cause) { if (api) return authError(cause); throw cause; }
|
||||
|
||||
const session = readSession(event.cookies.get(sessionCookie(event.url)));
|
||||
event.locals.user = session?.user??null;
|
||||
event.locals.sessionHash = session?.tokenHash??null;
|
||||
const setup = setupRequired();
|
||||
if (permission !== 'public') {
|
||||
if (!session) {
|
||||
if (api) return json({error:setup?'Complete administrator setup before using the management API.':'Sign in to continue.',setupRequired:setup},{status:401,headers:{'Cache-Control':'no-store'}});
|
||||
redirect(303,setup?'/setup':`/login?next=${encodeURIComponent(event.url.pathname+event.url.search)}`);
|
||||
}
|
||||
if (session.user.mustChangePassword && permission !== 'authenticated') {
|
||||
if (api) return json({error:'Change your temporary password from Account before continuing.',mustChangePassword:true},{status:403,headers:{'Cache-Control':'no-store'}});
|
||||
redirect(303,'/account');
|
||||
}
|
||||
if (!roleAllows(session.user.role,permission)) {
|
||||
const message = permission==='admin'?'Administrator access is required.':'Operator access is required for this action.';
|
||||
if (api) return json({error:message},{status:403,headers:{'Cache-Control':'no-store'}});
|
||||
error(403,message);
|
||||
}
|
||||
// Launch APIs can also save credentials. Operators may launch with saved
|
||||
// keys, but only administrators may introduce or replace server keys.
|
||||
if (session.user.role === 'operator' && method === 'POST' && ['/api/servers','/api/deployment-collections'].includes(routeId??'')) {
|
||||
try {
|
||||
const body = await authBody(event.request.clone(),1024*1024);
|
||||
if (body.plutoKey || (body.keys && typeof body.keys==='object' && Object.values(body.keys).some(Boolean))) {
|
||||
return json({error:'An administrator must save server keys before an operator launches these servers.'},{status:403});
|
||||
}
|
||||
} catch (cause) { return authError(cause); }
|
||||
}
|
||||
} else if (['/login','/setup'].includes(routeId??'')) {
|
||||
if (session) redirect(303,session.user.mustChangePassword?'/account':'/');
|
||||
if (routeId==='/setup'&&!setup) redirect(303,'/login');
|
||||
if (routeId==='/login'&&setup) redirect(303,'/setup');
|
||||
}
|
||||
|
||||
let response = await resolve(event);
|
||||
if (session && !['GET','HEAD','OPTIONS'].includes(method) && response.status < 400) {
|
||||
recordAudit(session.user.id,'request',routeId,{method,status:response.status,path:event.url.pathname});
|
||||
}
|
||||
if (session && response.headers.get('content-type')?.startsWith('text/event-stream')) {
|
||||
response = protectEventStream(response,session.tokenHash,permission,event.request.signal);
|
||||
}
|
||||
const headers = new Headers(response.headers);
|
||||
headers.set('Cache-Control','no-store');
|
||||
headers.set('Vary',[headers.get('Vary'),'Cookie'].filter(Boolean).join(', '));
|
||||
headers.set('Referrer-Policy','same-origin');
|
||||
headers.set('X-Content-Type-Options','nosniff');
|
||||
headers.set('X-Frame-Options','DENY');
|
||||
return new Response(response.body,{status:response.status,statusText:response.statusText,headers});
|
||||
};
|
||||
@@ -0,0 +1,49 @@
|
||||
import { error, isHttpError, json, type Cookies } from '@sveltejs/kit';
|
||||
import { AuthError, SESSION_COOKIE, type AuthResult } from './index.js';
|
||||
import type { AuthUser } from '../../auth.js';
|
||||
|
||||
export function sessionCookie(url: URL): string { return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev'; }
|
||||
export function requireUserTransport(url: URL): void {
|
||||
if (url.protocol === 'https:') return;
|
||||
if (process.env.EC_AUTH_ALLOW_HTTP === 'true' && ['localhost','127.0.0.1','[::1]','::1'].includes(url.hostname)) return;
|
||||
error(426, 'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.');
|
||||
}
|
||||
export function requireSameOrigin(request: Request, url: URL): void {
|
||||
if (request.headers.get('origin') !== url.origin || request.headers.get('sec-fetch-site') === 'cross-site') {
|
||||
error(403, 'This action requires a same-origin request.');
|
||||
}
|
||||
}
|
||||
export function sessionResponse(result: AuthResult, cookies: Cookies, url: URL) {
|
||||
cookies.set(sessionCookie(url), result.session.token, { path: '/', httpOnly: true, secure: url.protocol === 'https:',
|
||||
sameSite: 'lax', expires: new Date(result.session.expiresAt) });
|
||||
return json({ user: result.user }, { headers: { 'Cache-Control':'no-store' } });
|
||||
}
|
||||
export function requireUser(user: AuthUser | null): AuthUser {
|
||||
if (!user) throw new AuthError('Sign in to continue.',401);
|
||||
return user;
|
||||
}
|
||||
export async function authBody(request: Request, maxBytes = 8192): Promise<Record<string, unknown>> {
|
||||
if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json')) throw new AuthError('Send a JSON request.',415);
|
||||
if (Number(request.headers.get('content-length')) > maxBytes) throw new AuthError('Request is too large.',413);
|
||||
const reader = request.body?.getReader(); if (!reader) throw new AuthError('A JSON object is required.');
|
||||
const chunks: Uint8Array[] = []; let length = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read(); if (done) break;
|
||||
length += value.byteLength;
|
||||
if (length > maxBytes) { void reader.cancel().catch(() => {}); throw new AuthError('Request is too large.',413); }
|
||||
chunks.push(value);
|
||||
}
|
||||
} finally { reader.releaseLock(); }
|
||||
let body: unknown;
|
||||
try { body = JSON.parse(Buffer.concat(chunks,length).toString('utf8')); } catch { throw new AuthError('Invalid JSON.'); }
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) throw new AuthError('A JSON object is required.');
|
||||
return body as Record<string, unknown>;
|
||||
}
|
||||
export function authError(cause: unknown): Response {
|
||||
if (isHttpError(cause)) return json({ error: cause.body.message }, { status:cause.status, headers:{'Cache-Control':'no-store'} });
|
||||
const message = cause instanceof Error ? cause.message : 'Authentication request failed.';
|
||||
const status = cause instanceof AuthError ? cause.status : message.includes('busy') ? 429 : message.startsWith('Use a password') ? 400 : 500;
|
||||
return json({ error: status === 500 ? 'The authentication request could not be completed.' : message },
|
||||
{ status, headers:{'Cache-Control':'no-store',...(status===429?{'Retry-After':'900'}:{})} });
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureAuthSchema } from './schema.js';
|
||||
import { hashPassword, verifyPassword } from './passwords.js';
|
||||
import type { AuthUser, UserRole } from '../../auth.js';
|
||||
|
||||
export const SESSION_COOKIE = '__Host-ec_session';
|
||||
export const SESSION_LIFETIME_MS = 12 * 60 * 60_000;
|
||||
export const SESSION_IDLE_MS = 30 * 60_000;
|
||||
const digest = (value: string) => createHash('sha256').update(value).digest('hex');
|
||||
function database() { const handle = db(); ensureAuthSchema(handle); return handle; }
|
||||
type UserRow = { id: string; username: string; display_name: string; password_hash: string; role: UserRole; disabled: number;
|
||||
must_change_password: number; created_at: number; updated_at: number; last_login_at: number | null };
|
||||
export class AuthError extends Error {
|
||||
constructor(message: string, public status = 400) { super(message); }
|
||||
}
|
||||
function present(row: UserRow): AuthUser {
|
||||
return { id: row.id, username: row.username, displayName: row.display_name, role: row.role, disabled: !!row.disabled,
|
||||
mustChangePassword: !!row.must_change_password, createdAt: new Date(row.created_at).toISOString(),
|
||||
updatedAt: new Date(row.updated_at).toISOString(), lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString() };
|
||||
}
|
||||
function userRow(id: string) { return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined; }
|
||||
export function getUser(id: string): AuthUser | null { const row = userRow(id); return row ? present(row) : null; }
|
||||
export function listUsers(): AuthUser[] { return (database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[]).map(present); }
|
||||
export function setupRequired(): boolean { return !(database().prepare('SELECT 1 FROM auth_users LIMIT 1').get()); }
|
||||
function username(value: unknown): string {
|
||||
if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value)) throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.');
|
||||
return value.toLowerCase();
|
||||
}
|
||||
function displayName(value: unknown, fallback: string): string {
|
||||
const name = value === undefined ? fallback : value;
|
||||
if (typeof name !== 'string' || !name.trim() || name.length > 100 || /[\x00-\x1f\x7f]/.test(name)) throw new AuthError('Display name must contain 1–100 printable characters.');
|
||||
return name.trim();
|
||||
}
|
||||
function role(value: unknown): UserRole {
|
||||
if (!['admin','operator','viewer'].includes(value as string)) throw new AuthError('Choose administrator, operator or viewer.');
|
||||
return value as UserRole;
|
||||
}
|
||||
export function recordAudit(actorId: string | null, action: string, targetId: string | null = null, details: Record<string, unknown> = {}): void {
|
||||
database().prepare('INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)')
|
||||
.run(actorId, action, targetId, JSON.stringify(details), Date.now());
|
||||
}
|
||||
export function listAuthAudit() {
|
||||
return database().prepare(`SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt
|
||||
FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200`).all();
|
||||
}
|
||||
|
||||
/** Persistent fixed windows are consumed before hashing, including failed setup. */
|
||||
export function consumeAuthAttempt(clientAddress: string, account: string, kind = 'login'): void {
|
||||
const now = Date.now(); const windowMs = 15 * 60_000;
|
||||
const ipKey = digest(`${kind}:ip:${clientAddress}`);
|
||||
const accountKey = digest(`${kind}:account:${account.toLowerCase()}`);
|
||||
database().transaction(() => {
|
||||
database().prepare('DELETE FROM auth_rate_limits WHERE window_start < ?').run(now - 24 * 60 * 60_000);
|
||||
for (const [key, limit] of [[ipKey, 30], [accountKey, 10]] as const) {
|
||||
const previous = database().prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?').get(key) as { window_start: number; attempts: number } | undefined;
|
||||
if (previous && previous.window_start > now - windowMs && previous.attempts >= limit) throw new AuthError('Too many attempts. Try again in 15 minutes.', 429);
|
||||
}
|
||||
for (const key of [ipKey, accountKey]) database().prepare(`INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1)
|
||||
ON CONFLICT(key) DO UPDATE SET attempts=CASE WHEN window_start <= ? THEN 1 ELSE attempts+1 END,
|
||||
window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END`).run(key, now, now - windowMs, now - windowMs);
|
||||
}).immediate();
|
||||
}
|
||||
|
||||
function insertUser(input: { username: string; displayName: string; role: UserRole; passwordHash: string; mustChangePassword: boolean }): AuthUser {
|
||||
if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username)) throw new AuthError('That username is already in use.', 409);
|
||||
const id = randomUUID(); const now = Date.now();
|
||||
database().prepare(`INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`)
|
||||
.run(id,input.username,input.displayName,input.passwordHash,input.role,Number(input.mustChangePassword),now,now);
|
||||
return getUser(id)!;
|
||||
}
|
||||
export interface AuthResult { user: AuthUser; session: { token: string; expiresAt: number } }
|
||||
export async function bootstrapAdmin(input: Record<string, unknown>, clientAddress: string): Promise<AuthResult> {
|
||||
consumeAuthAttempt(clientAddress, 'bootstrap', 'setup');
|
||||
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
|
||||
const expected = process.env.EC_SETUP_TOKEN;
|
||||
if (!expected || expected.length < 32) throw new AuthError('The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.', 503);
|
||||
if (typeof input.token !== 'string' || !timingSafeEqual(Buffer.from(digest(input.token), 'hex'), Buffer.from(digest(expected), 'hex'))) throw new AuthError('Invalid setup token.', 401);
|
||||
const name = username(input.username); const label = displayName(input.displayName, name);
|
||||
const passwordHash = await hashPassword(input.password);
|
||||
return database().transaction(() => {
|
||||
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
|
||||
const user = insertUser({ username: name, displayName: label, passwordHash, role: 'admin', mustChangePassword: false });
|
||||
recordAudit(user.id, 'setup', user.id);
|
||||
return { user, session: createSession(user.id) };
|
||||
}).immediate();
|
||||
}
|
||||
export async function authenticatePassword(input: Record<string, unknown>, clientAddress: string): Promise<AuthResult> {
|
||||
const name = typeof input.username === 'string' ? input.username.slice(0,64).toLowerCase() : '';
|
||||
consumeAuthAttempt(clientAddress, name);
|
||||
const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as UserRow | undefined;
|
||||
const valid = await verifyPassword(input.password, row?.password_hash ?? null);
|
||||
const result = database().transaction(() => {
|
||||
const current = row ? userRow(row.id) : undefined;
|
||||
if (!valid || !current || current.disabled || current.password_hash !== row?.password_hash) {
|
||||
recordAudit(null, 'login.failed'); return null;
|
||||
}
|
||||
database().prepare('UPDATE auth_users SET last_login_at=? WHERE id=?').run(Date.now(), current.id);
|
||||
recordAudit(current.id, 'login', current.id);
|
||||
return { user: getUser(current.id)!, session: createSession(current.id) };
|
||||
}).immediate();
|
||||
if (!result) throw new AuthError('Invalid username or password.', 401);
|
||||
return result;
|
||||
}
|
||||
|
||||
export function createSession(userId: string): { token: string; expiresAt: number } {
|
||||
const now = Date.now(); const user = userRow(userId);
|
||||
if (!user || user.disabled) throw new AuthError('Account is unavailable.', 401);
|
||||
const token = randomBytes(32).toString('base64url'); const expiresAt = now + SESSION_LIFETIME_MS;
|
||||
database().prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?').run(now,now-SESSION_IDLE_MS);
|
||||
database().prepare('INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)').run(digest(token),userId,now,expiresAt,now);
|
||||
return { token, expiresAt };
|
||||
}
|
||||
export function readSession(token: string | undefined, touch = true): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
|
||||
if (!token || !/^[A-Za-z0-9_-]{43}$/.test(token)) return null;
|
||||
return readSessionHash(digest(token), touch);
|
||||
}
|
||||
export function readSessionHash(tokenHash: string, touch = false): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
|
||||
const now = Date.now();
|
||||
const row = database().prepare(`SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id
|
||||
WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0`).get(tokenHash,now,now-SESSION_IDLE_MS) as (UserRow & { expires_at: number; last_seen_at: number }) | undefined;
|
||||
if (!row) return null;
|
||||
if (touch && row.last_seen_at < now-60_000) database().prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?').run(now,tokenHash);
|
||||
return { user: present(row), tokenHash, expiresAt: row.expires_at };
|
||||
}
|
||||
export function revokeSession(token: string | undefined): void { if (token) database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token)); }
|
||||
export function revokeUserSessions(actorId: string, userId: string): void {
|
||||
database().transaction(() => {
|
||||
requireAdministrator(actorId);
|
||||
if (!userRow(userId)) throw new AuthError('User not found.',404);
|
||||
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
|
||||
recordAudit(actorId,'sessions.revoked',userId);
|
||||
})();
|
||||
}
|
||||
function requireAdministrator(actorId: string) {
|
||||
const actor = userRow(actorId);
|
||||
if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password) throw new AuthError('Administrator access is required.',403);
|
||||
}
|
||||
export async function createUser(actorId: string, input: Record<string, unknown>): Promise<AuthUser> {
|
||||
requireAdministrator(actorId);
|
||||
const name = username(input.username), label = displayName(input.displayName,name), userRole = role(input.role);
|
||||
const passwordHash = await hashPassword(input.password);
|
||||
return database().transaction(() => {
|
||||
requireAdministrator(actorId);
|
||||
const user = insertUser({ username:name,displayName:label,role:userRole,passwordHash,mustChangePassword:true });
|
||||
recordAudit(actorId,'user.created',user.id,{role:userRole}); return user;
|
||||
}).immediate();
|
||||
}
|
||||
export async function updateUser(actorId: string, id: string, input: Record<string, unknown>): Promise<AuthUser> {
|
||||
requireAdministrator(actorId);
|
||||
if (input.disabled !== undefined && typeof input.disabled !== 'boolean') throw new AuthError('Disabled must be true or false.');
|
||||
if (input.password !== undefined && actorId === id) throw new AuthError('Change your own password from Account using your current password.');
|
||||
const passwordHash = input.password === undefined ? null : await hashPassword(input.password);
|
||||
return database().transaction(() => {
|
||||
requireAdministrator(actorId);
|
||||
const current = userRow(id); if (!current) throw new AuthError('User not found.',404);
|
||||
const nextRole = input.role === undefined ? current.role : role(input.role);
|
||||
const disabled = input.disabled === undefined ? current.disabled : Number(input.disabled);
|
||||
const label = displayName(input.displayName,current.display_name);
|
||||
if (current.role === 'admin' && !current.disabled && (nextRole !== 'admin' || disabled)) {
|
||||
const other = database().prepare("SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1").get(id);
|
||||
if (!other) throw new AuthError('Keep at least one enabled administrator.',409);
|
||||
}
|
||||
database().prepare(`UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?`)
|
||||
.run(label,nextRole,disabled,passwordHash??current.password_hash,passwordHash?1:current.must_change_password,Date.now(),id);
|
||||
if (passwordHash || disabled !== current.disabled || nextRole !== current.role) database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id);
|
||||
recordAudit(actorId,'user.updated',id,{role:nextRole,disabled:!!disabled,passwordReset:!!passwordHash});
|
||||
return getUser(id)!;
|
||||
}).immediate();
|
||||
}
|
||||
export async function changePassword(userId: string, currentPassword: unknown, newPassword: unknown): Promise<AuthResult> {
|
||||
const original = userRow(userId);
|
||||
if (!original || original.disabled || !await verifyPassword(currentPassword,original.password_hash)) throw new AuthError('Current password is incorrect.',400);
|
||||
if (currentPassword === newPassword) throw new AuthError('Choose a different password.');
|
||||
const passwordHash = await hashPassword(newPassword);
|
||||
return database().transaction(() => {
|
||||
const current = userRow(userId);
|
||||
if (!current || current.disabled || current.password_hash !== original.password_hash) throw new AuthError('Account changed during password verification. Sign in again.',409);
|
||||
database().prepare('UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?').run(passwordHash,Date.now(),userId);
|
||||
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
|
||||
recordAudit(userId,'password.changed',userId);
|
||||
return { user: getUser(userId)!, session: createSession(userId) };
|
||||
}).immediate();
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { randomBytes, scrypt, timingSafeEqual } from 'node:crypto';
|
||||
|
||||
// OWASP's scrypt baseline: 128 MiB, r=8, p=1. Bound concurrent work so
|
||||
// unauthenticated password attempts cannot allocate unlimited memory.
|
||||
const parameters = { N: 131072, r: 8, p: 1, maxmem: 256 * 1024 * 1024 };
|
||||
let active = 0;
|
||||
|
||||
/** @param {unknown} password */
|
||||
export function validatePassword(password) {
|
||||
if (typeof password !== 'string' || [...password].length < 15 || [...password].length > 128) {
|
||||
throw new Error('Use a password or passphrase between 15 and 128 characters.');
|
||||
}
|
||||
return password;
|
||||
}
|
||||
|
||||
/** @param {string} password @param {string} salt @returns {Promise<Buffer>} */
|
||||
function derive(password, salt) {
|
||||
if (active >= 2) return Promise.reject(new Error('Password service is busy. Try again shortly.'));
|
||||
active++;
|
||||
return new Promise((resolve, reject) => {
|
||||
scrypt(password, salt, 64, parameters, (error, result) => {
|
||||
active--;
|
||||
if (error) reject(error); else resolve(result);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {unknown} password */
|
||||
export async function hashPassword(password) {
|
||||
const validated = validatePassword(password);
|
||||
const salt = randomBytes(32).toString('hex');
|
||||
const key = await derive(validated, salt);
|
||||
return `scrypt$131072$8$1$${salt}$${key.toString('hex')}`;
|
||||
}
|
||||
|
||||
/** @param {unknown} password @param {string | null} encoded */
|
||||
export async function verifyPassword(password, encoded) {
|
||||
if (typeof password !== 'string' || [...password].length > 128) return false;
|
||||
const parts = encoded?.split('$');
|
||||
const valid = parts?.length === 6 && parts[0] === 'scrypt' && parts[1] === '131072' && parts[2] === '8' && parts[3] === '1'
|
||||
&& /^[a-f0-9]{64}$/.test(parts[4]) && /^[a-f0-9]{128}$/.test(parts[5]);
|
||||
// Missing accounts perform the same expensive operation as real accounts.
|
||||
const result = await derive(password, valid ? parts[4] : '0'.repeat(64));
|
||||
return timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
const initialized = new WeakSet<Database.Database>();
|
||||
export function ensureAuthSchema(handle: Database.Database): void {
|
||||
if (initialized.has(handle)) return;
|
||||
handle.exec(`
|
||||
CREATE TABLE IF NOT EXISTS auth_users (
|
||||
id TEXT PRIMARY KEY, username TEXT NOT NULL COLLATE NOCASE UNIQUE,
|
||||
display_name TEXT NOT NULL, password_hash TEXT NOT NULL,
|
||||
role TEXT NOT NULL CHECK(role IN ('admin','operator','viewer')),
|
||||
disabled INTEGER NOT NULL DEFAULT 0, must_change_password INTEGER NOT NULL DEFAULT 0,
|
||||
created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, last_login_at INTEGER
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS auth_sessions (
|
||||
token_hash TEXT PRIMARY KEY, user_id TEXT NOT NULL REFERENCES auth_users(id) ON DELETE CASCADE,
|
||||
created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, last_seen_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS auth_sessions_user ON auth_sessions(user_id);
|
||||
CREATE TABLE IF NOT EXISTS auth_rate_limits (
|
||||
key TEXT PRIMARY KEY, window_start INTEGER NOT NULL, attempts INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS auth_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, actor_id TEXT, action TEXT NOT NULL,
|
||||
target_id TEXT, details TEXT NOT NULL, created_at INTEGER NOT NULL
|
||||
);
|
||||
`);
|
||||
initialized.add(handle);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { createHash, timingSafeEqual } from 'node:crypto';
|
||||
|
||||
export type TelemetryAuthentication =
|
||||
| { ok: true }
|
||||
| { ok: false; status: 401 | 503; error: string };
|
||||
|
||||
const tokenPattern = /^[A-Za-z0-9._~+\/-]+=*$/;
|
||||
const validToken = (value: string): boolean => value.length >= 32 && value.length <= 512 && tokenPattern.test(value);
|
||||
const hash = (value: string): Buffer => createHash('sha256').update(value, 'utf8').digest();
|
||||
|
||||
/** Machine telemetry has its own credential; browser sessions never authorize it. */
|
||||
export function authenticateTelemetry(request: Request): TelemetryAuthentication {
|
||||
const expected = process.env.EC_TELEMETRY_TOKEN ?? '';
|
||||
if (!validToken(expected)) {
|
||||
return { ok: false, status: 503, error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.' };
|
||||
}
|
||||
|
||||
const authorization = request.headers.get('authorization');
|
||||
const custom = request.headers.get('x-ec-telemetry-token');
|
||||
const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+\/-]+=*)$/i)?.[1];
|
||||
const denied = (): TelemetryAuthentication => ({ ok: false, status: 401, error: 'A valid game telemetry token is required.' });
|
||||
|
||||
// Reject malformed/ambiguous credentials rather than falling back to a
|
||||
// second header. A shared secret must never come from a URL, cookie or body.
|
||||
if (authorization !== null && !bearer) return denied();
|
||||
if (custom !== null && !validToken(custom)) return denied();
|
||||
if (bearer !== undefined && !validToken(bearer)) return denied();
|
||||
const supplied = bearer ?? custom;
|
||||
if (!supplied) return denied();
|
||||
const suppliedHash = hash(supplied);
|
||||
if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom))) return denied();
|
||||
if (!timingSafeEqual(suppliedHash, hash(expected))) return denied();
|
||||
return { ok: true };
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import * as fs from 'node:fs/promises';
|
||||
import { relative, resolve, sep } from 'node:path';
|
||||
import { config } from '../config.js';
|
||||
import { nativeInstance, nativeRequest } from './native.js';
|
||||
|
||||
function remote(path: string) {
|
||||
for (const root of [config.serversRoot, config.t6ServersRoot]) {
|
||||
if (!root) continue;
|
||||
const rel = relative(resolve(root), resolve(path));
|
||||
if (!rel || rel.startsWith('..') || rel.startsWith(sep)) continue;
|
||||
const [name, ...parts] = rel.split(sep);
|
||||
const instance = nativeInstance(name);
|
||||
if (instance) return { instance, path: parts.join('/') || '.' };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function missing(path: string): never { throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' }); }
|
||||
|
||||
export async function readFile(path: string): Promise<Buffer> {
|
||||
const target = remote(path);
|
||||
if (!target) return fs.readFile(path);
|
||||
const result = await nativeRequest<{ contentBase64: string | null }>(target.instance, 'files.read', { path: target.path });
|
||||
return result.contentBase64 === null ? missing(path) : Buffer.from(result.contentBase64, 'base64');
|
||||
}
|
||||
|
||||
export async function exists(path: string): Promise<boolean> {
|
||||
try { await stat(path); return true; }
|
||||
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; }
|
||||
}
|
||||
|
||||
export async function stat(path: string): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> {
|
||||
const target = remote(path);
|
||||
if (!target) return fs.stat(path);
|
||||
const result = await nativeRequest<{ exists?: boolean; size: number; mtime: string | number; isDirectory: boolean; isFile: boolean } | null>(target.instance, 'files.stat', { path: target.path });
|
||||
if (result === null || result.exists === false) return missing(path);
|
||||
return { size: result.size, mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime), isDirectory: () => result.isDirectory, isFile: () => result.isFile };
|
||||
}
|
||||
|
||||
export async function readdir(path: string, options: { withFileTypes: true }): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> {
|
||||
const target = remote(path);
|
||||
if (!target) return fs.readdir(path, options);
|
||||
const result = await nativeRequest<{ entries: { name: string; isDirectory: boolean; isFile?: boolean }[] }>(target.instance, 'files.list', { path: target.path });
|
||||
return result.entries.map(entry => ({ name: entry.name, isDirectory: () => entry.isDirectory, isFile: () => entry.isFile ?? !entry.isDirectory }));
|
||||
}
|
||||
|
||||
export async function nativeTail(path: string, lines: number) {
|
||||
const target = remote(path);
|
||||
if (!target) return undefined;
|
||||
return nativeRequest<import('../fleet/logs.js').TailResult | null>(target.instance, 'logs.read', { path: target.path, lines });
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { db } from '../db/index.js';
|
||||
import { getHost, queueHostOperation, waitHostOperation } from '../hosts/index.js';
|
||||
|
||||
export interface NativeInstance {
|
||||
id: string; server: string; host_id: string; generation: number; library_id: string;
|
||||
image: string; bind_address: string; port: number; public_address: string | null;
|
||||
public_port: number | null; created_at: string; retired_at: string | null;
|
||||
}
|
||||
|
||||
export function nativeInstance(server: string): NativeInstance | null {
|
||||
return db().prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL').get(server) as NativeInstance | undefined ?? null;
|
||||
}
|
||||
|
||||
export function nativeDeployment(id: string): boolean {
|
||||
return !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id = ?').get(id);
|
||||
}
|
||||
|
||||
/** Only the agent receives host paths, Docker access, or game credentials. */
|
||||
export async function nativeRequest<T = Record<string, unknown>>(instance: NativeInstance, kind: string, payload: Record<string, unknown> = {}, key?: string): Promise<T> {
|
||||
const host = getHost(instance.host_id);
|
||||
if (!host || host.connectionState !== 'online') throw new Error('Host agent is offline; instance state is unknown.');
|
||||
const operation = queueHostOperation(instance.host_id, kind, payload, {
|
||||
instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID()
|
||||
});
|
||||
const result = await waitHostOperation(operation.id, 25_000);
|
||||
if (result.state !== 'completed') throw new Error(result.error || `Agent operation ${result.id} is ${result.state}; its original ID is retained.`);
|
||||
// Do not apply replies from a retired generation to a replacement server.
|
||||
const current = nativeInstance(instance.server);
|
||||
if (current?.id !== instance.id || current.generation !== instance.generation) throw new Error('Instance was replaced during the operation.');
|
||||
return result.result as T;
|
||||
}
|
||||
|
||||
export async function nativeHealth(server: string): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> {
|
||||
const instance = nativeInstance(server);
|
||||
if (!instance) return { ok: false, reason: 'Native instance is not registered' };
|
||||
try {
|
||||
const result = await nativeRequest<{ state: string; healthy: boolean }>(instance, 'instance.inspect');
|
||||
return result.healthy ? { ok: true, state: result.state } : { ok: false, state: result.state, reason: `Container ${result.state}` };
|
||||
} catch (error) { return { ok: false, state: 'unknown', reason: (error as Error).message }; }
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
|
||||
export function migrateNativeInstances(handle: Database.Database): void {
|
||||
handle.exec(`
|
||||
CREATE TABLE IF NOT EXISTS native_instances (
|
||||
id TEXT PRIMARY KEY, server TEXT NOT NULL, host_id TEXT NOT NULL,
|
||||
generation INTEGER NOT NULL DEFAULT 1, library_id TEXT NOT NULL,
|
||||
image TEXT NOT NULL, bind_address TEXT NOT NULL DEFAULT '0.0.0.0',
|
||||
port INTEGER NOT NULL, public_address TEXT, public_port INTEGER,
|
||||
created_at TEXT NOT NULL, retired_at TEXT
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS native_instance_live_name ON native_instances(server) WHERE retired_at IS NULL;
|
||||
CREATE TABLE IF NOT EXISTS native_provisioning (
|
||||
deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
|
||||
instance_id TEXT NOT NULL REFERENCES native_instances(id), generation INTEGER NOT NULL,
|
||||
stage_operation_id TEXT, operation_id TEXT, files_json TEXT,
|
||||
applied_at TEXT, verified_at TEXT
|
||||
);
|
||||
`);
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { emptyMetrics } from '../fleet/beta-evaluation.js';
|
||||
import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js';
|
||||
import { initializeFleetSettings } from './fleet-settings.js';
|
||||
import { migrateServerHistory } from './server-history.js';
|
||||
import { migrateNativeInstances } from '../backends/schema.js';
|
||||
|
||||
let _db: Database.Database | null = null;
|
||||
|
||||
@@ -366,6 +367,7 @@ export function db(): Database.Database {
|
||||
]);
|
||||
backfillLeaderboard(handle);
|
||||
initializeFleetSettings(handle);
|
||||
migrateNativeInstances(handle);
|
||||
} catch (err) {
|
||||
handle.close();
|
||||
throw err;
|
||||
|
||||
@@ -10,8 +10,10 @@ async function exclusive<T>(id: number, action: () => Promise<T>): Promise<T> {
|
||||
try { return await action(); } finally { operations.delete(id); }
|
||||
}
|
||||
|
||||
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { readFile, exists } from '../backends/files.js';
|
||||
import { getHostOperation } from '../hosts/index.js';
|
||||
|
||||
import { join, posix } from 'node:path';
|
||||
import { betaArchiveOf, configFileOf, destOfKind, type ArtifactKind } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
@@ -204,6 +206,14 @@ export class BetaError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
function assertNativeActivationSettled(betaId: number): void {
|
||||
const pending = db().prepare(`SELECT n.operation_id FROM native_provisioning n
|
||||
JOIN provisioning_sources s ON s.operation_id=n.deployment_id
|
||||
WHERE s.kind='beta' AND s.source_id=? AND n.operation_id IS NOT NULL
|
||||
AND n.files_json IS NOT NULL AND n.applied_at IS NULL`).all(String(betaId)) as { operation_id: string }[];
|
||||
if (pending.some(row => getHostOperation(row.operation_id)?.state !== 'failed')) throw new BetaError('The host is still reconciling file activation. Wait for its durable before images before promoting or rolling back.', 'conflict');
|
||||
}
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
@@ -563,6 +573,7 @@ export async function rollbackBeta(betaId: number, options: { restart?: 'manual'
|
||||
return exclusive(betaId, () => rollbackBetaInternal(betaId, options));
|
||||
}
|
||||
async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual' | 'when-empty' }): Promise<RollbackResult> {
|
||||
assertNativeActivationSettled(betaId);
|
||||
const beta = getBeta(betaId);
|
||||
if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta');
|
||||
if (beta.state !== 'running' && beta.state !== 'ready-promote') {
|
||||
@@ -658,7 +669,7 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
|
||||
|
||||
// Wipe archive + close out beta.
|
||||
const archive = betaArchiveOf(betaId);
|
||||
if (existsSync(archive)) await rm(archive, { recursive: true, force: true });
|
||||
if (await exists(archive)) await rm(archive, { recursive: true, force: true });
|
||||
|
||||
// Per-version test-failed transition. Skip a version if (a) another beta
|
||||
// still gates it at ready-promote/promoted, or (b) it's the currently-
|
||||
@@ -764,6 +775,7 @@ export async function promoteBeta(betaId: number, reviewedToken?: string): Promi
|
||||
return exclusive(betaId, () => promoteBetaInternal(betaId, reviewedToken));
|
||||
}
|
||||
async function promoteBetaInternal(betaId: number, reviewedToken?: string): Promise<PromoteResult> {
|
||||
assertNativeActivationSettled(betaId);
|
||||
const test = storedPlan(betaId);
|
||||
if (test && test.outcome !== 'passed') throw new BetaError('Test plan has not passed', 'not-ready-promote');
|
||||
const beta = getBeta(betaId);
|
||||
@@ -842,7 +854,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
|
||||
// Non-beta matching-scope server: current on-disk file (if any)
|
||||
// is the "before" state for a promote-then-rollback round trip.
|
||||
const targetPath = join(dest, script.target_rel_path);
|
||||
if (existsSync(targetPath)) {
|
||||
if (await exists(targetPath)) {
|
||||
const priorContent = await readFile(targetPath);
|
||||
const inv = getServerScript(name, script.target_rel_path);
|
||||
plan = {
|
||||
@@ -906,13 +918,13 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
|
||||
hadFile = true;
|
||||
}
|
||||
}
|
||||
if (!betaServerSet.has(name) && existsSync(targetPath)) {
|
||||
if (!betaServerSet.has(name) && await exists(targetPath)) {
|
||||
// Capture an initial preview; activation records the actual before image.
|
||||
priorContent = await readFile(targetPath);
|
||||
priorSha = sha256Buffer(priorContent);
|
||||
hadFile = true;
|
||||
}
|
||||
const requiresActivation = !betaServerSet.has(name) || !existsSync(targetPath)
|
||||
const requiresActivation = !betaServerSet.has(name) || !await exists(targetPath)
|
||||
|| sha256Buffer(await readFile(targetPath)) !== composedSha;
|
||||
cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation });
|
||||
}
|
||||
@@ -1038,7 +1050,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
|
||||
db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId);
|
||||
|
||||
const archive = betaArchiveOf(betaId);
|
||||
if (existsSync(archive)) await rm(archive, { recursive: true, force: true });
|
||||
if (await exists(archive)) await rm(archive, { recursive: true, force: true });
|
||||
|
||||
const landed = Array.from(new Set([
|
||||
...plans.map((p) => p.server),
|
||||
|
||||
@@ -11,6 +11,8 @@ import { getServerKey } from './server-keys.js';
|
||||
import { serverObservation } from './status.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from './servers.js';
|
||||
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
|
||||
import { nativeSelection, nextNativePort } from '../provisioning/native.js';
|
||||
import { nativeInstance } from '../backends/native.js';
|
||||
|
||||
function coverage(entries: BlueprintEntry[]) {
|
||||
type Server = { name: string; game: string | null; map_slug: string | null; map: string | null };
|
||||
@@ -76,18 +78,25 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): C
|
||||
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
|
||||
const lobby = nextLobbyNumber(base, names);
|
||||
const name = lobby === null ? null : `${base}_${lobby}`;
|
||||
const port = nextAvailablePort(ports, releasingServers);
|
||||
const port = entry.recipe.native ? nextNativePort(entry.recipe.native, ports) : nextAvailablePort(ports, releasingServers);
|
||||
if (name) names.push(name);
|
||||
if (port !== null) ports.push(port);
|
||||
return { entryIndex, name, port, hasSavedKey: !!name && !!getServerKey(name) };
|
||||
}));
|
||||
}
|
||||
|
||||
function blockersFor(servers: CollectionPreview['servers']): string[] {
|
||||
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
|
||||
const blockers: string[] = [];
|
||||
if (!servers.length) return blockers;
|
||||
if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Configure Komodo credentials and a target host before launching.');
|
||||
if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
|
||||
if (entries.some(entry => entry.recipe.native)) {
|
||||
for (const entry of entries) {
|
||||
try { nativeSelection(entry.recipe.native as unknown as Record<string, unknown>, entry.recipe.game); }
|
||||
catch (error) { blockers.push((error as Error).message); }
|
||||
}
|
||||
} else {
|
||||
if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Select and prepare a host before launching.');
|
||||
if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
|
||||
}
|
||||
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
|
||||
if (servers.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
|
||||
return blockers;
|
||||
@@ -95,7 +104,7 @@ function blockersFor(servers: CollectionPreview['servers']): string[] {
|
||||
|
||||
function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
|
||||
const blockers: string[] = [];
|
||||
if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.');
|
||||
if (removals.some(server => !nativeInstance(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.');
|
||||
for (const server of removals) {
|
||||
const blocker = activeBetaRemovalBlocker(server.name);
|
||||
if (blocker) blockers.push(blocker);
|
||||
@@ -103,14 +112,21 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
|
||||
return blockers;
|
||||
}
|
||||
|
||||
export function previewCollection(blueprintId: unknown, mode: unknown = 'fill'): CollectionPreview {
|
||||
export function previewCollection(blueprintId: unknown, mode: unknown = 'fill', selection?: Record<string, unknown>): CollectionPreview {
|
||||
if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.');
|
||||
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
|
||||
return db().transaction((): CollectionPreview => {
|
||||
const { entries, blueprintName } = blueprintEntries(blueprintId);
|
||||
if (selection?.hostId !== undefined || !config.komodo.address) {
|
||||
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
|
||||
for (const entry of entries) {
|
||||
entry.recipe.native = nativeSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
|
||||
entry.recipe.image = entry.recipe.native.image!;
|
||||
}
|
||||
}
|
||||
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
|
||||
const servers = allocate(entries, removals.map(server => server.name));
|
||||
const blockers = [...blockersFor(servers), ...removalBlockersFor(removals)];
|
||||
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
|
||||
const id = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
||||
db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
|
||||
@@ -141,7 +157,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
const { retainedServers, removals } = collectionPlan(entries, preview.mode);
|
||||
const releasingServers = removals.map(server => server.name);
|
||||
const servers = allocate(entries, releasingServers);
|
||||
const blockers = [...blockersFor(servers), ...removalBlockersFor(removals)];
|
||||
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
|
||||
if (blockers.length) throw new Error(blockers.join(' '));
|
||||
if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) {
|
||||
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { readdir, readFile, stat } from 'node:fs/promises';
|
||||
import { readdir, readFile, stat } from '../backends/files.js';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
|
||||
import { join, posix, relative, sep } from 'node:path';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFile, stat } from 'node:fs/promises';
|
||||
import { readFile, stat } from '../backends/files.js';
|
||||
|
||||
export async function sha256(path: string): Promise<string> {
|
||||
const buf = await readFile(path);
|
||||
|
||||
@@ -9,6 +9,7 @@ import { getBaselineForScope, getVersion, listPromotedOverlaysForScope } from '.
|
||||
import { sha256Buffer } from './files.js';
|
||||
|
||||
export interface LaunchRecipe {
|
||||
native?: { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number };
|
||||
format: 1;
|
||||
game: ZombieGame;
|
||||
mapSlug: string;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { open, stat } from 'node:fs/promises';
|
||||
import { resolve, sep } from 'node:path';
|
||||
import { rootOfServer, gameOfServer } from '../config.js';
|
||||
import { nativeTail } from '../backends/files.js';
|
||||
|
||||
// Whitelisted log files exposed to the UI, mapped to their path relative
|
||||
// to each server's root directory. Adding a new supported log = one entry
|
||||
@@ -63,6 +64,8 @@ export async function tailLog(
|
||||
path: string,
|
||||
lines: number = DEFAULT_TAIL_LINES
|
||||
): Promise<TailResult | null> {
|
||||
const remote = await nativeTail(path, lines);
|
||||
if (remote !== undefined) return remote;
|
||||
let st;
|
||||
try {
|
||||
st = await stat(path);
|
||||
|
||||
@@ -3,6 +3,8 @@ import { existsSync } from 'node:fs';
|
||||
import { config, destOf, serverDir, rootOfServer } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { parseServerName } from './map-aliases.js';
|
||||
import { nativeInstance } from '../backends/native.js';
|
||||
import { stat as managedStat } from '../backends/files.js';
|
||||
|
||||
/**
|
||||
* All server folders under the game roots. Used by seedServerRegistry to backfill
|
||||
@@ -46,6 +48,10 @@ export async function ensureDest(
|
||||
create: boolean
|
||||
): Promise<{ ok: true; path: string } | { ok: false; reason: string }> {
|
||||
const dest = destOf(server);
|
||||
if (nativeInstance(server)) {
|
||||
try { return (await managedStat(dest)).isDirectory() ? { ok: true, path: dest } : { ok: false, reason: 'Agent script directory is unavailable' }; }
|
||||
catch (error) { return { ok: false, reason: (error as Error).message }; }
|
||||
}
|
||||
if (existsSync(dest)) {
|
||||
try {
|
||||
const s = await stat(dest);
|
||||
@@ -63,6 +69,10 @@ export async function ensureDest(
|
||||
// -- server registry --
|
||||
|
||||
export interface ServerRow {
|
||||
native_instance_id?: string | null;
|
||||
host_id?: string | null;
|
||||
instance_generation?: number | null;
|
||||
game_library_id?: string | null;
|
||||
name: string;
|
||||
komodo_type: 'deployment' | 'stack' | 'container';
|
||||
komodo_name: string;
|
||||
@@ -146,11 +156,13 @@ export async function seedServerRegistry(): Promise<number> {
|
||||
}
|
||||
|
||||
export function getServer(name: string): ServerRow | undefined {
|
||||
return db().prepare('SELECT * FROM servers WHERE name = ?').get(name) as ServerRow | undefined;
|
||||
return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
|
||||
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.name=?`).get(name) as ServerRow | undefined;
|
||||
}
|
||||
|
||||
export function listServers(): ServerRow[] {
|
||||
return db().prepare('SELECT * FROM servers WHERE destroyed_at IS NULL ORDER BY name').all() as ServerRow[];
|
||||
return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id
|
||||
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.destroyed_at IS NULL ORDER BY s.name`).all() as ServerRow[];
|
||||
}
|
||||
|
||||
/** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { stat } from 'node:fs/promises';
|
||||
import { Readable } from 'node:stream';
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { getLibraryFile } from './index.js';
|
||||
import { assertRegularPath, parseByteRange } from './storage.js';
|
||||
|
||||
export function gameFileError(cause: unknown): Response {
|
||||
const status = typeof cause === 'object' && cause !== null && 'status' in cause && typeof cause.status === 'number' ? cause.status : 400;
|
||||
return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.' }, { status });
|
||||
}
|
||||
export async function readGameJson(request: Request, maxBytes = 64 * 1024): Promise<Record<string, unknown>> {
|
||||
if (!request.body) throw new Error('A JSON request body is required.');
|
||||
const reader = request.body.getReader();
|
||||
const chunks: Uint8Array[] = []; let bytes = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const next = await reader.read(); if (next.done) break;
|
||||
bytes += next.value.byteLength;
|
||||
if (bytes > maxBytes) { await reader.cancel(); throw new Error('Request body exceeds its size limit.'); }
|
||||
chunks.push(next.value);
|
||||
}
|
||||
} finally { reader.releaseLock(); }
|
||||
const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) throw new Error('Expected a JSON object.');
|
||||
return body;
|
||||
}
|
||||
|
||||
export async function serveGameFile(request: Request, versionId: string, requestedPath: string): Promise<Response> {
|
||||
const file = getLibraryFile(versionId, requestedPath);
|
||||
if (!file || file.state !== 'ready') return json({ error: 'File is not in a published manifest.' }, { status: 404 });
|
||||
const path = await assertRegularPath(file.storageLocation, file.path);
|
||||
if ((await stat(path)).size !== file.size) return json({ error: 'Stored file failed size verification.' }, { status: 503 });
|
||||
const etag = `"${file.sha256}"`;
|
||||
const headers = new Headers({ 'Content-Type': 'application/octet-stream', 'Accept-Ranges': 'bytes', ETag: etag, 'Cache-Control': 'private, no-cache', 'X-Content-SHA256': file.sha256 });
|
||||
if (request.headers.get('if-none-match') === etag) return new Response(null, { status: 304, headers });
|
||||
let range;
|
||||
try { range = parseByteRange(request.headers.has('if-range') && request.headers.get('if-range') !== etag ? null : request.headers.get('range'), file.size); }
|
||||
catch { headers.set('Content-Range', `bytes */${file.size}`); return new Response(null, { status: 416, headers }); }
|
||||
headers.set('Content-Length', String(range ? range.end - range.start + 1 : file.size));
|
||||
if (range) headers.set('Content-Range', `bytes ${range.start}-${range.end}/${file.size}`);
|
||||
const body = request.method === 'HEAD' ? null : Readable.toWeb(createReadStream(path, range ?? undefined)) as ReadableStream<Uint8Array>;
|
||||
return new Response(body, { status: range ? 206 : 200, headers });
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { mkdir, rename, stat } from 'node:fs/promises';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { db } from '../db/index.js';
|
||||
import { config } from '../config.js';
|
||||
import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js';
|
||||
import { ensureGameFileSchema } from './schema.js';
|
||||
import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
|
||||
import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState } from '../../game-files.js';
|
||||
|
||||
function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
|
||||
export const gameFilesRoot = () => resolve(process.env.GAME_FILES_ROOT ?? join(dirname(config.dbPath), 'game-files'));
|
||||
const stagingRoot = (id: string) => join(gameFilesRoot(), '.imports', id);
|
||||
type VersionRow = { id: string; library_id: string; game: GameLibraryGame; version: string; description: string; dlc: string; state: GameLibraryVersion['state']; size_bytes: number; manifest: string; manifest_sha256: string | null; created_at: string; published_at: string | null; storage_location: string; source_host_id: string; source_volume: string; error: string | null; operation_id: string | null };
|
||||
type CacheRow = { host_id: string; version_id: string; status: LibraryCacheState; bytes_downloaded: number; bytes_total: number; manifest_sha256: string | null; error: string | null; updated_at: string; operation_id: string | null };
|
||||
const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id, libraryVersionId: row.version_id, status: row.status, bytesDownloaded: row.bytes_downloaded, bytesTotal: row.bytes_total, manifestSha256: row.manifest_sha256, error: row.error, updatedAt: row.updated_at, operationId: row.operation_id });
|
||||
function versionFromRow(row: VersionRow): GameLibraryVersion {
|
||||
const files = JSON.parse(row.manifest) as GameFileEntry[];
|
||||
return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
|
||||
description: row.description, dlc: JSON.parse(row.dlc), state: row.state, sizeBytes: row.size_bytes,
|
||||
files, fileCount: files.length, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
|
||||
publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id,
|
||||
sourceVolume: row.source_volume, error: row.error, operationId: row.operation_id,
|
||||
caches: (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
|
||||
}
|
||||
export function getLibraryVersion(id: string): GameLibraryVersion | null {
|
||||
const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined;
|
||||
return row ? versionFromRow(row) : null;
|
||||
}
|
||||
export function getLibraryFile(versionId: string, path: string) {
|
||||
validateGamePath(path);
|
||||
return database().prepare(`SELECT f.path,f.size,f.sha256,v.state,v.source_host_id AS sourceHostId,v.storage_location AS storageLocation
|
||||
FROM game_file_entries f JOIN game_file_versions v ON v.id=f.version_id WHERE f.version_id=? AND f.path=?`)
|
||||
.get(versionId, path) as (GameFileEntry & { state: GameLibraryVersion['state']; sourceHostId: string; storageLocation: string }) | undefined ?? null;
|
||||
}
|
||||
export function newestReadyLibrary(game: string): GameLibraryVersion | null {
|
||||
const row = database().prepare("SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as VersionRow | undefined;
|
||||
return row ? versionFromRow(row) : null;
|
||||
}
|
||||
export function listLibraries(): GameLibrary[] {
|
||||
reconcileLibraryOperations();
|
||||
return (database().prepare('SELECT * FROM game_file_libraries ORDER BY name').all() as Array<{ id: string; name: string; game: GameLibraryGame; description: string; created_at: string }>).map(row => ({
|
||||
id: row.id, name: row.name, game: row.game, description: row.description, createdAt: row.created_at,
|
||||
versions: (database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.id=? ORDER BY v.created_at DESC, v.rowid DESC').all(row.id) as VersionRow[]).map(versionFromRow)
|
||||
}));
|
||||
}
|
||||
export function getHostLibraryCache(hostId: string, versionId: string): HostLibraryCache | null {
|
||||
const row = database().prepare('SELECT * FROM host_library_caches WHERE host_id=? AND version_id=?').get(hostId, versionId) as CacheRow | undefined;
|
||||
return row ? cacheFromRow(row) : null;
|
||||
}
|
||||
export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion {
|
||||
const version = getLibraryVersion(versionId);
|
||||
if (!version || version.state !== 'ready') throw new Error('Select a published game-file version.');
|
||||
const cache = getHostLibraryCache(hostId, versionId);
|
||||
if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifestSha256 !== version.manifestSha256) throw new Error('The selected host must finish downloading and verifying this game-file version.');
|
||||
return version;
|
||||
}
|
||||
|
||||
function textField(input: unknown, label: string, max: number, required = true): string {
|
||||
if (typeof input !== 'string' || input.length > max || (required && !input.trim())) throw new Error(`Provide a valid ${label}.`);
|
||||
return input.trim();
|
||||
}
|
||||
export function createLibraryImport(input: Record<string, unknown>): GameLibraryVersion {
|
||||
const game = input.game;
|
||||
if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.');
|
||||
const hostId = textField(input.hostId, 'source host', 128);
|
||||
const host = getHost(hostId);
|
||||
if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled source host.');
|
||||
const name = textField(input.name, 'library name', 160);
|
||||
const description = textField(input.description ?? '', 'description', 4000, false);
|
||||
const version = textField(input.version, 'version label', 120);
|
||||
const dlcInput = input.dlc ?? [];
|
||||
if (!Array.isArray(dlcInput) || dlcInput.length > 64) throw new Error('DLC must be a list of content names.');
|
||||
const dlc = dlcInput.map(value => textField(value, 'DLC name', 120));
|
||||
const sourceVolume = game === 't5' ? 't5_game_files' : 't6_game_files';
|
||||
const handle = database();
|
||||
let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
|
||||
if (library && library.game !== game) throw new Error('This library name is already used for another game.');
|
||||
const id = randomUUID(); const now = new Date().toISOString();
|
||||
handle.transaction(() => {
|
||||
if (!library) {
|
||||
library = { id: randomUUID(), game };
|
||||
handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(library.id, name, game, description, now);
|
||||
}
|
||||
if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
|
||||
handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume)
|
||||
VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, JSON.stringify(dlc), now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
|
||||
})();
|
||||
try {
|
||||
const operation = queueHostOperation(hostId, 'library.import', {
|
||||
importId: id, game, sourceVolume, manifestUrl: `/api/agent/imports/${id}`,
|
||||
uploadBaseUrl: `/api/agent/imports/${id}/files`, completeUrl: `/api/agent/imports/${id}/complete`
|
||||
}, { idempotencyKey: `library-import:${id}` });
|
||||
handle.prepare('UPDATE game_file_versions SET operation_id=? WHERE id=?').run(operation.id, id);
|
||||
} catch (cause) {
|
||||
handle.prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=?").run((cause as Error).message, id);
|
||||
throw cause;
|
||||
}
|
||||
return getLibraryVersion(id)!;
|
||||
}
|
||||
|
||||
export function requireImportOwner(hostId: string, id: string): GameLibraryVersion {
|
||||
const version = getLibraryVersion(id);
|
||||
if (!version || version.sourceHostId !== hostId) throw new Error('Import not found for this host.');
|
||||
return version;
|
||||
}
|
||||
export function setImportManifest(hostId: string, id: string, input: unknown): GameLibraryVersion {
|
||||
const version = requireImportOwner(hostId, id);
|
||||
const files = validateManifest(input); const checksum = manifestChecksum(files);
|
||||
if (version.manifestSha256) {
|
||||
if (version.manifestSha256 !== checksum) throw new Error('An import manifest cannot change. Start a new version.');
|
||||
return version;
|
||||
}
|
||||
if (version.state !== 'importing') throw new Error('This import is not writable.');
|
||||
database().transaction(() => {
|
||||
database().prepare('UPDATE game_file_versions SET manifest=?,manifest_sha256=?,size_bytes=? WHERE id=? AND state=\'importing\'').run(JSON.stringify(files), checksum, files.reduce((sum, file) => sum + file.size, 0), id);
|
||||
const insert = database().prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)');
|
||||
for (const file of files) insert.run(id, file.path, file.size, file.sha256);
|
||||
})();
|
||||
return getLibraryVersion(id)!;
|
||||
}
|
||||
export async function uploadImportFile(hostId: string, id: string, path: string, body: ReadableStream<Uint8Array> | null): Promise<void> {
|
||||
const file = getLibraryFile(id, path);
|
||||
if (!file || file.sourceHostId !== hostId) throw new Error('File is not listed in a manifest belonging to this host.');
|
||||
if (file.state !== 'importing' || publishing.has(id)) throw new Error('This import is not writable.');
|
||||
await receiveGameFile(stagingRoot(id), file, body);
|
||||
}
|
||||
|
||||
const publishing = new Map<string, Promise<GameLibraryVersion>>();
|
||||
export async function publishImport(hostId: string, id: string): Promise<GameLibraryVersion> {
|
||||
const version = requireImportOwner(hostId, id);
|
||||
if (version.state === 'ready') return version;
|
||||
if (version.state !== 'importing' || !version.files.length || !version.manifestSha256) throw new Error('Import needs a complete manifest before publication.');
|
||||
const existing = publishing.get(id); if (existing) return existing;
|
||||
const task = (async () => {
|
||||
let root = stagingRoot(id);
|
||||
// A restart between the atomic directory rename and metadata commit is recoverable.
|
||||
try { if ((await stat(version.storageLocation)).isDirectory()) root = version.storageLocation; }
|
||||
catch (cause) { if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; }
|
||||
for (const file of version.files) await verifyGameFile(root, file);
|
||||
if (root !== version.storageLocation) {
|
||||
await mkdir(dirname(version.storageLocation), { recursive: true });
|
||||
await rename(root, version.storageLocation);
|
||||
}
|
||||
database().prepare("UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'").run(new Date().toISOString(), id);
|
||||
return getLibraryVersion(id)!;
|
||||
})();
|
||||
publishing.set(id, task);
|
||||
try { return await task; } finally { publishing.delete(id); }
|
||||
}
|
||||
|
||||
/** Large libraries can take minutes to verify. Keep HTTP requests bounded while the source host polls. */
|
||||
export function importPublicationStatus(hostId: string, id: string) {
|
||||
const version = requireImportOwner(hostId, id);
|
||||
return {
|
||||
status: version.state === 'ready' ? 'published' as const : version.state === 'failed' ? 'failed' as const : publishing.has(id) ? 'verifying' as const : 'pending' as const,
|
||||
version: { id: version.id, game: version.game, sizeBytes: version.sizeBytes, manifestSha256: version.manifestSha256 },
|
||||
error: version.error
|
||||
};
|
||||
}
|
||||
export function requestImportPublication(hostId: string, id: string) {
|
||||
const version = requireImportOwner(hostId, id);
|
||||
if (version.state === 'importing' && !publishing.has(id)) {
|
||||
if (!version.files.length || !version.manifestSha256) throw new Error('Import requires a manifest before publication.');
|
||||
void publishImport(hostId, id).catch(cause => {
|
||||
database().prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'")
|
||||
.run((cause instanceof Error ? cause.message : 'Library publication failed.').slice(0, 2000), id);
|
||||
});
|
||||
}
|
||||
return importPublicationStatus(hostId, id);
|
||||
}
|
||||
|
||||
export function updateHostLibraryCaches(hostId: string, input: unknown, options: { snapshot?: boolean } = {}): void {
|
||||
if (!Array.isArray(input) || input.length > 2000) throw new Error('Invalid host library report.');
|
||||
const handle = database();
|
||||
const now = new Date().toISOString();
|
||||
handle.transaction(() => {
|
||||
const reported = new Set<string>();
|
||||
for (const raw of input) {
|
||||
if (!raw || typeof raw !== 'object') throw new Error('Invalid library cache report.');
|
||||
const id = raw.libraryVersionId ?? raw.id;
|
||||
if (typeof id !== 'string') throw new Error('Invalid library version ID.');
|
||||
if (reported.has(id)) throw new Error('Duplicate version in host cache report.');
|
||||
reported.add(id);
|
||||
const version = getLibraryVersion(id); if (!version || version.state !== 'ready') continue;
|
||||
let status = raw.status ?? raw.state;
|
||||
if (!['missing', 'downloading', 'verifying', 'ready', 'failed', 'outdated'].includes(status)) throw new Error('Invalid library cache state.');
|
||||
const checksum = typeof raw.manifestSha256 === 'string' ? raw.manifestSha256 : null;
|
||||
let error = typeof raw.error === 'string' ? raw.error.slice(0, 2000) : null;
|
||||
if (['ready', 'outdated'].includes(status) && checksum !== version.manifestSha256) { status = 'failed'; error = 'Host manifest checksum does not match the published version.'; }
|
||||
const bytes = raw.bytesDownloaded ?? (['ready', 'outdated'].includes(status) ? version.sizeBytes : 0);
|
||||
if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > version.sizeBytes) throw new Error('Invalid library download progress.');
|
||||
if (['ready', 'outdated'].includes(status) && bytes !== version.sizeBytes) { status = 'failed'; error = 'Host reported an incomplete library as verified.'; }
|
||||
handle.prepare(`INSERT INTO host_library_caches(host_id,version_id,status,bytes_downloaded,bytes_total,manifest_sha256,error,updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(host_id,version_id) DO UPDATE SET status=excluded.status,
|
||||
bytes_downloaded=excluded.bytes_downloaded,bytes_total=excluded.bytes_total,manifest_sha256=excluded.manifest_sha256,error=excluded.error,updated_at=excluded.updated_at`)
|
||||
.run(hostId, id, status, bytes, version.sizeBytes, checksum, error, now);
|
||||
}
|
||||
if (options.snapshot) {
|
||||
const cached = handle.prepare('SELECT version_id FROM host_library_caches WHERE host_id=?').all(hostId) as { version_id: string }[];
|
||||
for (const cache of cached) if (!reported.has(cache.version_id)) {
|
||||
handle.prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=? AND version_id=?")
|
||||
.run(now, hostId, cache.version_id);
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
export function queueLibraryDownload(hostId: string, versionId: string) {
|
||||
const version = getLibraryVersion(versionId);
|
||||
if (!version || version.state !== 'ready') throw new Error('Choose a published library version.');
|
||||
const host = getHost(hostId);
|
||||
if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled host.');
|
||||
if (Number(host.inventory.storageAvailable ?? 0) < version.sizeBytes + host.minimumFreeBytes) throw new Error('This host has insufficient storage for this library and its minimum free-space reserve.');
|
||||
const prior = getHostLibraryCache(hostId, versionId);
|
||||
if (prior?.operationId) {
|
||||
const operation = getHostOperation(prior.operationId);
|
||||
if (operation && ['pending', 'accepted', 'running', 'unknown'].includes(operation.state)) return operation;
|
||||
}
|
||||
const operation = queueHostOperation(hostId, 'library.download', {
|
||||
libraryVersionId: versionId, game: version.game, manifestUrl: `/api/agent/libraries/${versionId}/manifest`,
|
||||
filesBaseUrl: `/api/agent/libraries/${versionId}/files`, statusUrl: `/api/agent/libraries/${versionId}/status`,
|
||||
manifestSha256: version.manifestSha256, sizeBytes: version.sizeBytes
|
||||
});
|
||||
updateHostLibraryCaches(hostId, [{ libraryVersionId: versionId, status: 'downloading', bytesDownloaded: 0 }]);
|
||||
database().prepare('UPDATE host_library_caches SET operation_id=? WHERE host_id=? AND version_id=?').run(operation.id, hostId, versionId);
|
||||
return operation;
|
||||
}
|
||||
|
||||
export function reconcileLibraryOperations(): void {
|
||||
const handle = database();
|
||||
for (const row of handle.prepare("SELECT id,operation_id FROM game_file_versions WHERE state='importing' AND operation_id IS NOT NULL").all() as { id: string; operation_id: string }[]) {
|
||||
const operation = getHostOperation(row.operation_id);
|
||||
if (operation?.state === 'failed') handle.prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'").run(operation.error ?? 'Host import failed.', row.id);
|
||||
}
|
||||
for (const cache of handle.prepare("SELECT * FROM host_library_caches WHERE status IN ('downloading','verifying') AND operation_id IS NOT NULL").all() as CacheRow[]) {
|
||||
const operation = getHostOperation(cache.operation_id!);
|
||||
if (operation?.state === 'failed') handle.prepare("UPDATE host_library_caches SET status='failed',error=?,updated_at=? WHERE host_id=? AND version_id=?").run(operation.error ?? 'Host download failed.', new Date().toISOString(), cache.host_id, cache.version_id);
|
||||
}
|
||||
}
|
||||
|
||||
/** Lightweight list used in launch forms; large manifests stay on the detail endpoint. */
|
||||
export function libraryOverview() {
|
||||
const hosts = listHosts();
|
||||
return { libraries: listLibraries().map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({
|
||||
...version, caches: hosts.map(host => {
|
||||
const cached = getHostLibraryCache(host.id, version.id);
|
||||
const latest = library.versions.find(item => item.state === 'ready');
|
||||
if (cached) return { ...cached, status: cached.status === 'ready' && latest?.id !== version.id ? 'outdated' as const : cached.status };
|
||||
return {
|
||||
hostId: host.id, libraryVersionId: version.id, status: 'missing' as const, bytesDownloaded: 0,
|
||||
bytesTotal: version.sizeBytes, manifestSha256: null, error: null, updatedAt: version.createdAt, operationId: null
|
||||
};
|
||||
})
|
||||
})) })), hosts };
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
|
||||
const initialized = new WeakSet<Database.Database>();
|
||||
export function ensureGameFileSchema(handle: Database.Database): void {
|
||||
if (initialized.has(handle)) return;
|
||||
handle.exec(`
|
||||
CREATE TABLE IF NOT EXISTS game_file_libraries (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL COLLATE NOCASE UNIQUE,
|
||||
game TEXT NOT NULL CHECK(game IN ('t5','t6')), description TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS game_file_versions (
|
||||
id TEXT PRIMARY KEY, library_id TEXT NOT NULL REFERENCES game_file_libraries(id),
|
||||
version TEXT NOT NULL, description TEXT NOT NULL, dlc TEXT NOT NULL,
|
||||
state TEXT NOT NULL CHECK(state IN ('importing','ready','failed')),
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0, manifest TEXT NOT NULL DEFAULT '[]',
|
||||
manifest_sha256 TEXT, created_at TEXT NOT NULL, published_at TEXT,
|
||||
storage_location TEXT NOT NULL, source_host_id TEXT NOT NULL, source_volume TEXT NOT NULL,
|
||||
error TEXT, operation_id TEXT, UNIQUE(library_id, version)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS host_library_caches (
|
||||
host_id TEXT NOT NULL, version_id TEXT NOT NULL REFERENCES game_file_versions(id),
|
||||
status TEXT NOT NULL CHECK(status IN ('missing','downloading','verifying','ready','failed','outdated')),
|
||||
bytes_downloaded INTEGER NOT NULL DEFAULT 0, bytes_total INTEGER NOT NULL DEFAULT 0,
|
||||
manifest_sha256 TEXT, error TEXT, updated_at TEXT NOT NULL, operation_id TEXT,
|
||||
PRIMARY KEY(host_id, version_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS game_file_entries (
|
||||
version_id TEXT NOT NULL REFERENCES game_file_versions(id), path TEXT NOT NULL,
|
||||
size INTEGER NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY(version_id,path)
|
||||
);
|
||||
INSERT OR IGNORE INTO game_file_entries(version_id,path,size,sha256)
|
||||
SELECT v.id,json_extract(f.value,'$.path'),json_extract(f.value,'$.size'),json_extract(f.value,'$.sha256')
|
||||
FROM game_file_versions v,json_each(v.manifest) f;
|
||||
CREATE TRIGGER IF NOT EXISTS game_file_versions_immutable BEFORE UPDATE ON game_file_versions
|
||||
WHEN OLD.state = 'ready' BEGIN SELECT RAISE(ABORT, 'Published game-file versions are immutable'); END;
|
||||
`);
|
||||
initialized.add(handle);
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createReadStream, createWriteStream } from 'node:fs';
|
||||
import { mkdir, lstat, rename, rm, stat } from 'node:fs/promises';
|
||||
import { dirname, isAbsolute, resolve, sep } from 'node:path';
|
||||
import { Readable, Transform } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { GameFileEntry } from '../../game-files.js';
|
||||
|
||||
export function validateGamePath(input: unknown): string {
|
||||
if (typeof input !== 'string' || !input || input.length > 1024 || /[\\\x00-\x1f\x7f:]/.test(input) || Buffer.from(input).toString('utf8') !== input
|
||||
|| isAbsolute(input) || input.split('/').some(part => !part || part === '.' || part === '..' || /[. ]$/.test(part))) {
|
||||
throw new Error('Game file paths must be relative, normalized paths without traversal.');
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
export function validateManifest(input: unknown): GameFileEntry[] {
|
||||
if (!Array.isArray(input) || !input.length || input.length > 200_000) throw new Error('Provide a manifest containing 1–200000 files.');
|
||||
const paths = new Set<string>();
|
||||
let total = 0;
|
||||
const files = input.map(raw => {
|
||||
if (!raw || typeof raw !== 'object') throw new Error('Invalid manifest entry.');
|
||||
const path = validateGamePath(raw.path);
|
||||
if (!Number.isSafeInteger(raw.size) || raw.size < 0 || typeof raw.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(raw.sha256)) throw new Error(`Invalid size or SHA256 for ${path}.`);
|
||||
const key = path.toLowerCase();
|
||||
if (paths.has(key)) throw new Error(`Duplicate game file path: ${path}.`);
|
||||
paths.add(key); total += raw.size;
|
||||
if (!Number.isSafeInteger(total)) throw new Error('Library size exceeds the supported limit.');
|
||||
return { path, size: raw.size as number, sha256: raw.sha256 as string };
|
||||
}).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
||||
for (const file of files) {
|
||||
const parts = file.path.toLowerCase().split('/');
|
||||
parts.pop();
|
||||
while (parts.length) {
|
||||
if (paths.has(parts.join('/'))) throw new Error('A manifest path cannot be both a file and a directory.');
|
||||
parts.pop();
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
export const manifestChecksum = (files: GameFileEntry[]): string => createHash('sha256').update(JSON.stringify(files)).digest('hex');
|
||||
|
||||
export function gameFilePath(root: string, path: string): string {
|
||||
const base = resolve(root);
|
||||
const result = resolve(base, validateGamePath(path));
|
||||
if (!result.startsWith(base + sep)) throw new Error('Invalid game file path.');
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Refuse symlinks at every existing component, including the managed root. */
|
||||
export async function assertRegularPath(root: string, path: string, allowMissing = false): Promise<string> {
|
||||
const result = gameFilePath(root, path);
|
||||
let current = resolve(root);
|
||||
const components = [current, ...validateGamePath(path).split('/').map(part => current = resolve(current, part))];
|
||||
for (let i = 0; i < components.length; i++) {
|
||||
let info;
|
||||
try { info = await lstat(components[i]); }
|
||||
catch (cause) { if (allowMissing && (cause as NodeJS.ErrnoException).code === 'ENOENT') continue; throw cause; }
|
||||
if (info.isSymbolicLink() || (i < components.length - 1 ? !info.isDirectory() : !info.isFile())) throw new Error('Game storage contains a non-regular path.');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function verifyGameFile(root: string, file: GameFileEntry): Promise<void> {
|
||||
const path = await assertRegularPath(root, file.path);
|
||||
const info = await stat(path);
|
||||
if (info.size !== file.size) throw new Error(`Size verification failed for ${file.path}.`);
|
||||
const hash = createHash('sha256');
|
||||
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
||||
if (hash.digest('hex') !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`);
|
||||
}
|
||||
|
||||
/** Stream to a private temporary file; publish only after byte count and SHA256 match. */
|
||||
export async function receiveGameFile(root: string, file: GameFileEntry, body: ReadableStream<Uint8Array> | null): Promise<void> {
|
||||
if (!body) throw new Error('A file body is required.');
|
||||
const path = await assertRegularPath(root, file.path, true);
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
const temp = `${path}.upload-${randomUUID()}`;
|
||||
let size = 0;
|
||||
const hash = createHash('sha256');
|
||||
const verifier = new Transform({ transform(chunk, _encoding, done) {
|
||||
size += chunk.length;
|
||||
if (size > file.size) return done(new Error('Upload exceeds the manifest size.'));
|
||||
hash.update(chunk); done(null, chunk);
|
||||
} });
|
||||
try {
|
||||
await pipeline(Readable.fromWeb(body as never), verifier, createWriteStream(temp, { flags: 'wx', mode: 0o600, flush: true }));
|
||||
if (size !== file.size || hash.digest('hex') !== file.sha256) throw new Error(`Upload checksum or size mismatch for ${file.path}.`);
|
||||
await rename(temp, path);
|
||||
} finally { await rm(temp, { force: true }); }
|
||||
}
|
||||
|
||||
export function parseByteRange(header: string | null, size: number): { start: number; end: number } | null {
|
||||
if (!header) return null;
|
||||
const match = /^bytes=(\d*)-(\d*)$/.exec(header);
|
||||
if (!match || (!match[1] && !match[2]) || size === 0) throw new Error('Unsatisfiable range.');
|
||||
let start = match[1] ? Number(match[1]) : Math.max(0, size - Number(match[2]));
|
||||
let end = match[1] && match[2] ? Number(match[2]) : size - 1;
|
||||
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || start >= size || end < start || (!match[1] && Number(match[2]) === 0)) throw new Error('Unsatisfiable range.');
|
||||
return { start, end: Math.min(end, size - 1) };
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { error, json } from '@sveltejs/kit';
|
||||
import type { Host } from '../../hosts.js';
|
||||
import { authenticateCredential } from './index.js';
|
||||
|
||||
export function requireAgentTransport(request: Request): void {
|
||||
if (new URL(request.url).protocol !== 'https:' && process.env.EC_AGENT_ALLOW_HTTP !== 'true') error(426, 'Host agents require HTTPS.');
|
||||
}
|
||||
export function authenticateAgent(request: Request): Host {
|
||||
requireAgentTransport(request);
|
||||
const authorization = request.headers.get('authorization');
|
||||
const token = authorization?.match(/^Bearer ([A-Za-z0-9_-]+)$/i)?.[1];
|
||||
const host = token ? authenticateCredential(token) : null;
|
||||
if (!host) error(401, 'A valid host credential is required.');
|
||||
return host;
|
||||
}
|
||||
export function requireManagementOrigin(request: Request): void {
|
||||
const origin = request.headers.get('origin');
|
||||
if ((origin && origin !== new URL(request.url).origin) || request.headers.get('sec-fetch-site') === 'cross-site') error(403, 'Cross-origin host management requests are not allowed.');
|
||||
// Agent credentials only authorize agent endpoints, never host management.
|
||||
if (request.headers.get('authorization')?.match(/^Bearer ec_(?:host|enroll)_/i)) error(403, 'Agent credentials cannot manage the controller.');
|
||||
}
|
||||
export async function objectBody(request: Request, maxBytes = 1024 * 1024): Promise<Record<string, unknown>> {
|
||||
if (Number(request.headers.get('content-length')) > maxBytes) error(413, 'Request is too large.');
|
||||
const reader = request.body?.getReader();
|
||||
if (!reader) error(400, 'Request body must be valid JSON.');
|
||||
const chunks: Uint8Array[] = []; let received = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
received += value.byteLength;
|
||||
if (received > maxBytes) { void reader.cancel().catch(() => {}); error(413, 'Request is too large.'); }
|
||||
chunks.push(value);
|
||||
}
|
||||
} finally { reader.releaseLock(); }
|
||||
const raw = Buffer.concat(chunks, received).toString('utf8');
|
||||
let body: unknown;
|
||||
try { body = JSON.parse(raw); } catch { error(400, 'Request body must be valid JSON.'); }
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) error(400, 'Request body must be an object.');
|
||||
return body as Record<string, unknown>;
|
||||
}
|
||||
export function hostApiError(cause: unknown): Response {
|
||||
if (cause && typeof cause === 'object' && 'status' in cause && 'body' in cause) throw cause;
|
||||
return json({ error: cause instanceof Error ? cause.message : 'Host request failed.' }, { status: 400 });
|
||||
}
|
||||
export function installerCommand(controller: string, token: string): string {
|
||||
const origin = new URL(controller).origin;
|
||||
const quote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'";
|
||||
return `curl --fail --silent --show-error --proto '=https' ${quote(origin + '/api/agent/install.sh')} | sudo bash -s -- --controller ${quote(origin)} --token ${quote(token)}`;
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { isIP } from 'node:net';
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureHostSchema } from './schema.js';
|
||||
import type { Host, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
|
||||
|
||||
export const AGENT_PROTOCOL_VERSION = 1;
|
||||
export const HEARTBEAT_MAX_AGE_MS = 90_000;
|
||||
const now = () => new Date().toISOString();
|
||||
const digest = (token: string) => createHash('sha256').update(token).digest('hex');
|
||||
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
|
||||
type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
|
||||
const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`;
|
||||
function hostFromRow(row: HostRow): Host {
|
||||
const inventory: HostInventory = JSON.parse(row.inventory);
|
||||
const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
|
||||
const reasons: string[] = [];
|
||||
if (!row.enrolled_at) reasons.push('Agent has not enrolled.');
|
||||
else if (!row.credential_active) reasons.push('Agent credential has been revoked.');
|
||||
if (!connected) reasons.push('Agent heartbeat is unavailable or stale.');
|
||||
if (inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) reasons.push('A compatible agent protocol is required.');
|
||||
if (!inventory.dockerRunning) reasons.push('Docker is not ready.');
|
||||
if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.');
|
||||
if (typeof inventory.storageAvailable !== 'number' || inventory.storageAvailable < row.minimum_free_bytes) reasons.push('Insufficient available game storage.');
|
||||
const infrastructureReady = reasons.length === 0;
|
||||
if (!inventory.runtimeImages?.length) reasons.push('Prepare a Plutainer runtime image.');
|
||||
if (!inventory.libraries?.some(library => library.state === 'ready')) reasons.push('Download and verify a game-file library.');
|
||||
if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
|
||||
if (row.mode === 'disabled') reasons.push('Host is disabled.');
|
||||
return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
|
||||
bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
|
||||
createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
|
||||
readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at || (infrastructureReady && reasons.length) ? 'preparing' : reasons.length ? 'degraded' : 'ready',
|
||||
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active };
|
||||
}
|
||||
export function listHosts(): Host[] { return (database().prepare(hostSql + ' ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
|
||||
export function getHost(id: string): Host | null {
|
||||
const row = database().prepare(hostSql + ' WHERE h.id = ?').get(id) as HostRow | undefined;
|
||||
return row ? hostFromRow(row) : null;
|
||||
}
|
||||
function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; }
|
||||
function textField(value: unknown, label: string, max = 200): string {
|
||||
if (typeof value !== 'string' || !value.trim() || value.length > max || /[\x00-\x1f]/.test(value)) throw new Error(`${label} must be a nonempty string up to ${max} characters.`);
|
||||
return value.trim();
|
||||
}
|
||||
function port(value: unknown): number { if (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 65535) throw new Error('Ports must be whole numbers between 1 and 65535.'); return value; }
|
||||
export function createHost(input: Record<string, unknown>): Host {
|
||||
const id = randomUUID(); const name = textField(input.name, 'Host name');
|
||||
const min = port(input.portMin ?? 28960); const max = port(input.portMax ?? 28980);
|
||||
if (max < min) throw new Error('The last game port must not precede the first.');
|
||||
const bind = input.bindAddress === undefined ? '0.0.0.0' : textField(input.bindAddress, 'Bind address');
|
||||
if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.');
|
||||
database().prepare('INSERT INTO ec_hosts(id,name,port_min,port_max,bind_address,created_at) VALUES(?,?,?,?,?,?)').run(id, name, min, max, bind, now());
|
||||
return requireHost(id);
|
||||
}
|
||||
export function updateHost(id: string, input: Record<string, unknown>): Host {
|
||||
const host = requireHost(id);
|
||||
const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
|
||||
const mode = input.mode ?? host.mode;
|
||||
if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
|
||||
const min = port(input.portMin ?? host.portMin); const max = port(input.portMax ?? host.portMax);
|
||||
if (max < min) throw new Error('The last game port must not precede the first.');
|
||||
const bind = input.bindAddress === undefined ? host.bindAddress : textField(input.bindAddress, 'Bind address');
|
||||
if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.');
|
||||
const publicAddress = input.publicAddress === undefined ? host.publicAddress : input.publicAddress === '' ? '' : textField(input.publicAddress, 'Public address');
|
||||
const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes;
|
||||
if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) throw new Error('Minimum free storage must be a nonnegative byte count.');
|
||||
database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
|
||||
return requireHost(id);
|
||||
}
|
||||
|
||||
export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } {
|
||||
requireHost(hostId);
|
||||
if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.');
|
||||
const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + ttlMs).toISOString();
|
||||
database().transaction(() => {
|
||||
database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(now(), hostId);
|
||||
database().prepare('INSERT INTO ec_host_enrollments(id,host_id,token_hash,created_at,expires_at) VALUES(?,?,?,?,?)').run(id, hostId, digest(token), now(), expiresAt);
|
||||
})();
|
||||
return { id, hostId, token, expiresAt };
|
||||
}
|
||||
function issueCredential(hostId: string, credential = 'ec_host_' + randomBytes(32).toString('base64url')): string {
|
||||
database().prepare('INSERT INTO ec_host_credentials(id,host_id,token_hash,created_at) VALUES(?,?,?,?)').run(randomUUID(), hostId, digest(credential), now());
|
||||
return credential;
|
||||
}
|
||||
export function validateInventory(value: unknown): HostInventory {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Host inventory must be an object.');
|
||||
const inventory = value as HostInventory;
|
||||
if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.');
|
||||
for (const key of ['hostname','agentVersion','os','architecture','cpu','dockerVersion']) {
|
||||
if (inventory[key] !== undefined && (typeof inventory[key] !== 'string' || String(inventory[key]).length > 1000)) throw new Error(`Invalid inventory ${key}.`);
|
||||
}
|
||||
for (const key of ['cpuCount','memoryTotal','memoryAvailable','storageTotal','storageAvailable','protocolVersion']) {
|
||||
if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`);
|
||||
}
|
||||
for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
|
||||
if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
|
||||
if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library =>
|
||||
!library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.');
|
||||
return inventory;
|
||||
}
|
||||
export function enrollHost(token: unknown, rawInventory: unknown): { hostId: string; credential: string; heartbeatSeconds: number } {
|
||||
if (typeof token !== 'string' || token.length > 200) throw new Error('Enrollment token is invalid or expired.');
|
||||
const inventory = validateInventory(rawInventory);
|
||||
return database().transaction(() => {
|
||||
const enrollment = database().prepare('SELECT id,host_id FROM ec_host_enrollments WHERE token_hash=? AND consumed_at IS NULL AND expires_at>?').get(digest(token), now()) as { id: string; host_id: string } | undefined;
|
||||
if (!enrollment) throw new Error('Enrollment token is invalid or expired.');
|
||||
const changed = database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE id=? AND consumed_at IS NULL').run(now(), enrollment.id);
|
||||
if (changed.changes !== 1) throw new Error('Enrollment token has already been consumed.');
|
||||
revokeCredentials(enrollment.host_id);
|
||||
const credential = issueCredential(enrollment.host_id);
|
||||
database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id);
|
||||
return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 };
|
||||
})();
|
||||
}
|
||||
export function authenticateCredential(token: string): Host | null {
|
||||
if (!token || token.length > 200) return null;
|
||||
const row = database().prepare('SELECT id,host_id FROM ec_host_credentials WHERE token_hash=? AND revoked_at IS NULL').get(digest(token)) as { id: string; host_id: string } | undefined;
|
||||
if (!row) return null;
|
||||
database().prepare('UPDATE ec_host_credentials SET last_used_at=? WHERE id=?').run(now(), row.id);
|
||||
return getHost(row.host_id);
|
||||
}
|
||||
export function revokeCredentials(hostId: string): void {
|
||||
requireHost(hostId);
|
||||
database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId);
|
||||
}
|
||||
export function rotateCredential(hostId: string, nextCredential?: string): string {
|
||||
requireHost(hostId);
|
||||
if (nextCredential !== undefined && !/^ec_host_[A-Za-z0-9_-]{43}$/.test(nextCredential)) throw new Error('New credentials must contain 32 random bytes encoded as base64url.');
|
||||
return database().transaction(() => {
|
||||
if (nextCredential) {
|
||||
const existing = database().prepare('SELECT host_id,revoked_at FROM ec_host_credentials WHERE token_hash=?').get(digest(nextCredential)) as { host_id: string; revoked_at: string | null } | undefined;
|
||||
if (existing) {
|
||||
if (existing.host_id !== hostId || existing.revoked_at) throw new Error('This credential cannot be reused.');
|
||||
return nextCredential;
|
||||
}
|
||||
}
|
||||
revokeCredentials(hostId); return issueCredential(hostId, nextCredential);
|
||||
})();
|
||||
}
|
||||
export function heartbeat(hostId: string, value: unknown): Host {
|
||||
requireHost(hostId); const inventory = validateInventory(value);
|
||||
database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), inventory.hostname ?? '', JSON.stringify(inventory), hostId);
|
||||
return requireHost(hostId);
|
||||
}
|
||||
export function requireReadyHost(hostId: string, options: { libraryId?: string; requiredBytes?: number; image?: string; port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): Host {
|
||||
const host = requireHost(hostId); const reasons = [...host.readinessReasons];
|
||||
if (options.requiredBytes !== undefined && (host.inventory.storageAvailable ?? 0) < options.requiredBytes + host.minimumFreeBytes) reasons.push('Insufficient storage for this deployment.');
|
||||
if (options.libraryId && !host.inventory.libraries?.some(library => library.id === options.libraryId && library.state === 'ready')) reasons.push('The selected game library is not ready on this host.');
|
||||
if (options.image && !host.inventory.runtimeImages?.includes(options.image)) reasons.push('The selected runtime image is not cached on this host.');
|
||||
if (options.port !== undefined && !hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol)) reasons.push('The selected host port is not available.');
|
||||
if (reasons.length) throw new Error(reasons.join(' '));
|
||||
return host;
|
||||
}
|
||||
|
||||
type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string };
|
||||
function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
|
||||
export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; }
|
||||
export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
|
||||
export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation {
|
||||
const host = requireHost(hostId);
|
||||
if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
|
||||
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
|
||||
if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
|
||||
return database().transaction(() => {
|
||||
if (options.idempotencyKey) {
|
||||
const existing = database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? AND idempotency_key=?').get(hostId, options.idempotencyKey) as OperationRow | undefined;
|
||||
if (existing) {
|
||||
if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || existing.payload !== JSON.stringify(payload)) throw new Error('Operation idempotency key is already associated with a different request.');
|
||||
return operationFromRow(existing);
|
||||
}
|
||||
}
|
||||
if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
|
||||
if (host.mode === 'disabled') throw new Error('Host is disabled. Enable it before requesting operations.');
|
||||
const id = options.id ?? randomUUID(); const timestamp = now();
|
||||
database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, JSON.stringify(payload), timestamp, timestamp);
|
||||
database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
|
||||
return getHostOperation(id)!;
|
||||
})();
|
||||
}
|
||||
export function pendingHostOperations(hostId: string): HostOperation[] {
|
||||
const host = requireHost(hostId);
|
||||
if (host.mode === 'disabled' || !host.connected) return [];
|
||||
// Accepted/running/unknown commands keep the same immutable ID. The agent
|
||||
// consults its durable journal before acknowledging or executing a replay.
|
||||
return database().transaction(() => {
|
||||
const rows = database().prepare(`SELECT * FROM ec_host_operations WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
|
||||
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
|
||||
ORDER BY COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`).all(hostId, Number(host.readiness === 'ready')) as OperationRow[];
|
||||
const deliveredAt = now();
|
||||
for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
|
||||
return rows.map(operationFromRow);
|
||||
})();
|
||||
}
|
||||
export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
|
||||
return database().transaction(() => {
|
||||
const operation = getHostOperation(id);
|
||||
if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
|
||||
if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.');
|
||||
if (['completed','failed'].includes(operation.state)) return operation;
|
||||
const state = input.state as HostOperationState;
|
||||
if (operation.state === 'running' && state === 'accepted') return operation;
|
||||
const error = input.error === undefined || input.error === null ? null : String(input.error).slice(0, 4000);
|
||||
const result = input.result === undefined ? null : JSON.stringify(input.result);
|
||||
if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
|
||||
database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id);
|
||||
if (operation.state !== state) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error);
|
||||
return getHostOperation(id)!;
|
||||
})();
|
||||
}
|
||||
export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (true) {
|
||||
const operation = getHostOperation(id);
|
||||
if (!operation) throw new Error('Host operation was not found.');
|
||||
if (['completed','failed','unknown'].includes(operation.state)) return operation;
|
||||
if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`);
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
}
|
||||
}
|
||||
|
||||
export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean {
|
||||
const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress;
|
||||
if (candidate < host.portMin || candidate > host.portMax) return false;
|
||||
const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[];
|
||||
return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address)));
|
||||
}
|
||||
export function nextHostPort(hostId: string, additionalPorts: number[] = [], bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp'): number | null {
|
||||
const host = requireHost(hostId);
|
||||
for (let candidate = host.portMin; candidate <= host.portMax; candidate++) if (!additionalPorts.includes(candidate) && hostPortAvailable(hostId, candidate, bindAddress, protocol)) return candidate;
|
||||
return null;
|
||||
}
|
||||
export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number {
|
||||
const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp';
|
||||
if (!isIP(bind)) throw new Error('Bind address must be an IP address.');
|
||||
if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.');
|
||||
if (!instanceId) throw new Error('An immutable instance ID is required.');
|
||||
return database().transaction(() => {
|
||||
const existing = database().prepare('SELECT port FROM ec_host_ports WHERE host_id=? AND instance_id=? AND bind_address=? AND protocol=?').get(hostId, instanceId, bind, protocol) as { port: number } | undefined;
|
||||
if (existing && (options.port === undefined || options.port === existing.port)) return existing.port;
|
||||
const allocated = options.port ?? nextHostPort(hostId, [], bind, protocol);
|
||||
if (allocated === null || !hostPortAvailable(hostId, allocated, bind, protocol, instanceId)) throw new Error('No available game port on the selected host.');
|
||||
database().prepare('INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) VALUES(?,?,?,?,?,?)').run(hostId, instanceId, bind, protocol, allocated, now());
|
||||
return allocated;
|
||||
})();
|
||||
}
|
||||
export function releaseHostPorts(instanceId: string): void { database().prepare('DELETE FROM ec_host_ports WHERE instance_id=?').run(instanceId); }
|
||||
@@ -0,0 +1,11 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
/** adapter-node copies public assets to build/client; Vite serves static in development. */
|
||||
export async function agentReleaseAsset(file: 'install.sh' | 'ec-agent.py' | 'manifest.json'): Promise<string> {
|
||||
try { return await readFile(resolve('build/client/agent', file), 'utf8'); }
|
||||
catch (cause) {
|
||||
if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause;
|
||||
return readFile(resolve('static/agent', file), 'utf8');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
|
||||
const initialized = new WeakSet<Database.Database>();
|
||||
export function ensureHostSchema(handle: Database.Database): void {
|
||||
if (initialized.has(handle)) return;
|
||||
handle.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ec_hosts (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL, hostname TEXT NOT NULL DEFAULT '',
|
||||
mode TEXT NOT NULL DEFAULT 'active' CHECK (mode IN ('active','draining','disabled')),
|
||||
port_min INTEGER NOT NULL DEFAULT 28960, port_max INTEGER NOT NULL DEFAULT 28980,
|
||||
bind_address TEXT NOT NULL DEFAULT '0.0.0.0', public_address TEXT NOT NULL DEFAULT '',
|
||||
minimum_free_bytes INTEGER NOT NULL DEFAULT 1073741824,
|
||||
created_at TEXT NOT NULL, enrolled_at TEXT, last_heartbeat TEXT, inventory TEXT NOT NULL DEFAULT '{}'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_enrollments (
|
||||
id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
|
||||
created_at TEXT NOT NULL, expires_at TEXT NOT NULL, consumed_at TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_credentials (
|
||||
id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
|
||||
created_at TEXT NOT NULL, revoked_at TEXT, last_used_at TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_operations (
|
||||
id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), kind TEXT NOT NULL,
|
||||
instance_id TEXT, generation INTEGER, idempotency_key TEXT,
|
||||
payload TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (state IN ('pending','accepted','running','completed','failed','unknown')),
|
||||
result TEXT, error TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, last_delivered_at TEXT,
|
||||
UNIQUE(host_id, idempotency_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ec_host_operations_queue ON ec_host_operations(host_id, state, created_at);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_operation_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, operation_id TEXT NOT NULL REFERENCES ec_host_operations(id),
|
||||
state TEXT NOT NULL, created_at TEXT NOT NULL, error TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_ports (
|
||||
host_id TEXT NOT NULL REFERENCES ec_hosts(id), instance_id TEXT NOT NULL,
|
||||
bind_address TEXT NOT NULL, protocol TEXT NOT NULL CHECK (protocol IN ('udp','tcp')),
|
||||
port INTEGER NOT NULL CHECK (port BETWEEN 1 AND 65535), created_at TEXT NOT NULL,
|
||||
PRIMARY KEY(host_id,bind_address,protocol,port)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
|
||||
`);
|
||||
const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
|
||||
if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
|
||||
initialized.add(handle);
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { config, komodoConfigured } from '../config.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { nativeHealth, nativeInstance } from '../backends/native.js';
|
||||
|
||||
// Ports `komodo_restart` in push_scripts.sh:622. Three shapes depending on
|
||||
// the tracked komodo_type: deployment | container | stack.
|
||||
@@ -353,6 +354,7 @@ async function fetchJsonWithSnippet(
|
||||
* the container endpoint isn't available.
|
||||
*/
|
||||
export async function komodoInspectHealthy(serverName: string): Promise<KomodoHealthResult> {
|
||||
if (getServer(serverName)?.native_instance_id) return nativeHealth(serverName);
|
||||
if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' };
|
||||
const row = getServer(serverName);
|
||||
if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' };
|
||||
|
||||
@@ -5,6 +5,7 @@ import { getServer, setServerField } from '../fleet/servers.js';
|
||||
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
|
||||
import { komodoResolveGamePort } from './komodo.js';
|
||||
import { parsePlutoniumRevision } from '../../plutonium.js';
|
||||
import { nativeInstance, nativeRequest } from '../backends/native.js';
|
||||
|
||||
// Plutonium/CoD RCON is UDP with a 4-byte 0xff prefix. Requests look like
|
||||
// \xff\xff\xff\xffrcon <password> <command>
|
||||
@@ -373,6 +374,26 @@ export function formatRawReply(buf: Buffer): string {
|
||||
* rather than just "unreachable".
|
||||
*/
|
||||
export async function rconStatusResult(server: string): Promise<RconStatusResult> {
|
||||
const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null;
|
||||
if (native) {
|
||||
const key = JSON.stringify(['agent', native.id, native.generation, latestGameSubmission(server)]);
|
||||
const existing = statusInFlight.get(key);
|
||||
if (existing) return existing;
|
||||
const endpoint = { host: `agent:${native.host_id}`, port: native.port };
|
||||
const request = (async (): Promise<RconStatusResult> => {
|
||||
try {
|
||||
const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'status' });
|
||||
const players = parsePlayerCount(reply.text);
|
||||
if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: reply.text.slice(0, 4000) };
|
||||
const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text);
|
||||
return { ok: true, endpoint, status: { players,
|
||||
capacity: positiveCapacity(info.sv_maxclients ?? header.sv_maxclients),
|
||||
map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null,
|
||||
gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null, roster: parsePlayerRoster(reply.text) }, rawReply: reply.text.slice(0, 4000) };
|
||||
} catch (error) { return { ok: false, endpoint, reason: 'timeout', detail: (error as Error).message }; }
|
||||
})().finally(() => statusInFlight.delete(key));
|
||||
statusInFlight.set(key, request); return request;
|
||||
}
|
||||
// Snapshot before any network work. A probe begun after a restart or a
|
||||
// replacement registration must not join one observing the old process.
|
||||
const createdAt = getServer(server)?.created_at;
|
||||
@@ -476,6 +497,14 @@ function positiveCapacity(value: string | undefined): number | null {
|
||||
|
||||
/** Query the running process, not the updater's on-disk files. */
|
||||
export async function rconPlutoniumVersion(server: string): Promise<{ revision: number | null; error: string | null }> {
|
||||
const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null;
|
||||
if (native) {
|
||||
try {
|
||||
const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'version' });
|
||||
const revision = parsePlutoniumRevision(reply.text);
|
||||
return { revision, error: revision === null ? 'Agent RCON reply did not identify a runtime revision.' : null };
|
||||
} catch (error) { return { revision: null, error: (error as Error).message }; }
|
||||
}
|
||||
const { endpoint } = await endpointFor(server);
|
||||
if (!endpoint) return { revision: null, error: 'No RCON endpoint available.' };
|
||||
const response = await sendRcon(endpoint.host, endpoint.port, 'version');
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured } from '../config.js';
|
||||
import { nativeInstance } from '../backends/native.js';
|
||||
import { getServer, listServers } from '../fleet/servers.js';
|
||||
import { hasActiveProvisioning } from '../fleet/provisioning.js';
|
||||
import { komodoInspectHealthy } from '../integrations/komodo.js';
|
||||
@@ -295,7 +296,7 @@ async function run(forceScan: boolean): Promise<void> {
|
||||
const lastScan = setting('watchdog_last_scan');
|
||||
if (!forceScan && lastScan && Date.now() - Date.parse(lastScan) < WATCHDOG_SCAN_MS) return;
|
||||
setSetting('watchdog_last_scan', iso());
|
||||
if (!komodoConfigured()) {
|
||||
if (!komodoConfigured() && !listServers().some(server => nativeInstance(server.name))) {
|
||||
setSetting('watchdog_scan_error', 'Configure Komodo before automatic recovery. RCON or fresh telemetry verifies game status.'); return;
|
||||
}
|
||||
const errors: string[] = [];
|
||||
|
||||
@@ -6,12 +6,14 @@ import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail,
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
|
||||
import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
|
||||
import { bindNativeOperation } from './native.js';
|
||||
|
||||
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
|
||||
const id = randomUUID();
|
||||
db().transaction(() => {
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
||||
VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
|
||||
bindNativeOperation(id, server);
|
||||
recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested',
|
||||
label: action === 'create' ? 'Server creation requested'
|
||||
: action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested` } });
|
||||
@@ -243,10 +245,10 @@ export function listDeployments() {
|
||||
|
||||
export function interruptDeployments(): void {
|
||||
db().transaction(() => {
|
||||
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[];
|
||||
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)").all() as { id: string }[];
|
||||
const finishedAt = new Date().toISOString();
|
||||
const error = 'The application restarted before this operation finished. Check the server before retrying.';
|
||||
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running'`).run(finishedAt, error);
|
||||
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)`).run(finishedAt, error);
|
||||
for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome',
|
||||
message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } });
|
||||
})();
|
||||
|
||||
@@ -28,6 +28,8 @@ import { getVersion } from '../fleet/scripts.js';
|
||||
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
||||
import { recordProgress, type ProgressUpdate } from './progress.js';
|
||||
import { checkRelease, freshRelease, probeVersion, release } from '../integrations/plutonium.js';
|
||||
import { nativeInstance, nativeDeployment } from '../backends/native.js';
|
||||
import { previewNativeCreation, queueNativeCreation, queueNativeRebuild, cancelQueuedNativeCreation } from './native.js';
|
||||
|
||||
export type CreateServerFailure =
|
||||
| 'bad-request'
|
||||
@@ -49,6 +51,12 @@ export type DestroyServerResult =
|
||||
| { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
|
||||
|
||||
export interface CreateServerInput {
|
||||
hostId?: string;
|
||||
libraryId?: string;
|
||||
image?: string;
|
||||
bindAddress?: string;
|
||||
publicAddress?: string;
|
||||
publicPort?: number;
|
||||
game?: ZombieGame; // omitted by older clients: T5
|
||||
gameMode: GameMode; // 'zombie' — only value supported today
|
||||
mapSlug: string; // 'cosmodrome'
|
||||
@@ -167,7 +175,8 @@ interface PreviewRow {
|
||||
}
|
||||
|
||||
/** Read-only checks; a preview never reserves a slot or writes credentials. */
|
||||
export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown }): LaunchPreview {
|
||||
export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown; hostId?: unknown; libraryId?: unknown }): LaunchPreview {
|
||||
if (input.hostId !== undefined || !config.komodo.address) return previewNativeCreation(input);
|
||||
if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.');
|
||||
let recipe: LaunchRecipe;
|
||||
let blueprintName: string | null = null;
|
||||
@@ -208,6 +217,8 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue
|
||||
|
||||
// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
|
||||
export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
|
||||
const nativePreview = input.previewId ? db().prepare("SELECT 1 FROM server_launch_previews WHERE id=? AND json_extract(recipe,'$.native.hostId') IS NOT NULL").get(input.previewId) : null;
|
||||
if (input.hostId || nativePreview || !config.komodo.address) return queueNativeCreation(input);
|
||||
return db().transaction((): CreateServerResult => {
|
||||
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
|
||||
if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };
|
||||
@@ -243,6 +254,7 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
|
||||
|
||||
/** Reserve the same identity and port before removing any part of the instance. */
|
||||
export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServerResult {
|
||||
if (nativeInstance(name)) return queueNativeRebuild(name, triggeredBy);
|
||||
const blocker = activeBetaRemovalBlocker(name);
|
||||
if (blocker) return { ok: false, kind: 'conflict', message: blocker };
|
||||
const server = getServer(name);
|
||||
@@ -535,6 +547,8 @@ function hasActiveCreation(name: string): boolean {
|
||||
}
|
||||
|
||||
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
|
||||
const cancelled = cancelQueuedNativeCreation(name, triggeredBy);
|
||||
if (cancelled) return { ok: true, warnings: [], deploymentId: cancelled };
|
||||
const blocker = activeBetaRemovalBlocker(name);
|
||||
if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild')
|
||||
@@ -597,6 +611,7 @@ async function runDestructions(): Promise<void> {
|
||||
}
|
||||
|
||||
async function processDestruction(operation: DestructionJob): Promise<void> {
|
||||
if (nativeDeployment(operation.id)) return;
|
||||
if (creating || destroying.has(operation.server)) return;
|
||||
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) return;
|
||||
destroying.add(operation.server);
|
||||
@@ -859,6 +874,7 @@ async function runCreations(): Promise<CreationOutcomes> {
|
||||
// deployment. A known interrupted startup must retain its readiness gate.
|
||||
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
|
||||
for (const { id } of interrupted) {
|
||||
if (nativeDeployment(id)) continue;
|
||||
if (resumeVerification(id)) continue;
|
||||
// A queued removal does not make the interrupted container disappear.
|
||||
// Keep its slot until removal finishes, without resuming normal work or
|
||||
@@ -883,6 +899,7 @@ async function runCreations(): Promise<CreationOutcomes> {
|
||||
WHERE d.server_deleted_at IS NULL AND d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
|
||||
for (const job of checking) {
|
||||
const id = job.deployment_id;
|
||||
if (nativeDeployment(id)) continue;
|
||||
if (getServer(job.server)?.game_port !== job.port) {
|
||||
finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
|
||||
continue;
|
||||
@@ -980,6 +997,7 @@ async function runCreations(): Promise<CreationOutcomes> {
|
||||
const queue = creationQueue();
|
||||
const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length);
|
||||
const eligible = queue.queued.filter(({ id }) => {
|
||||
if (nativeDeployment(id)) return false;
|
||||
const blocker = creationBlocker(id);
|
||||
if (blocker) deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker });
|
||||
else recordProgress(id, { phase: 'startup-queue', blocker: { code: available ? 'startup-queue' : 'startup-capacity',
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
import { randomBytes, randomUUID } from 'node:crypto';
|
||||
import { posix } from 'node:path';
|
||||
import { db } from '../db/index.js';
|
||||
import { config, relDestForKind, serverDir } from '../config.js';
|
||||
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
|
||||
import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js';
|
||||
import { getLibraryVersion, requireReadyLibrary } from '../game-files/index.js';
|
||||
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
|
||||
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
|
||||
import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
|
||||
import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
|
||||
import { getServerScript, recordScript, removeScript } from '../fleet/inventory.js';
|
||||
import { sha256Buffer } from '../fleet/files.js';
|
||||
import type { StagedFileChange } from '../fleet/staged-files.js';
|
||||
import type { LaunchPreview } from '../../deployment-plan.js';
|
||||
import type { ProvisioningAction } from '../../provisioning.js';
|
||||
|
||||
export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }
|
||||
type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
|
||||
interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
|
||||
instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null }
|
||||
interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
|
||||
|
||||
export function nativeSelection(input: Record<string, unknown>, game: string): NativeSelection {
|
||||
if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.');
|
||||
if (typeof input.libraryId !== 'string' || !input.libraryId) throw new Error('Select a published game-file version.');
|
||||
const version = getLibraryVersion(input.libraryId);
|
||||
if (!version || version.game !== game || version.state !== 'ready') throw new Error('Choose a published game-file version for the selected game.');
|
||||
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
|
||||
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
|
||||
const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 });
|
||||
requireReadyLibrary(host.id, version.id);
|
||||
const bindAddress = input.bindAddress === undefined || input.bindAddress === '' ? host.bindAddress : input.bindAddress;
|
||||
if (typeof bindAddress !== 'string') throw new Error('Invalid bind address.');
|
||||
if (input.publicAddress !== undefined && (typeof input.publicAddress !== 'string' || input.publicAddress.length > 253)) throw new Error('Invalid public address.');
|
||||
if (input.publicPort !== undefined && (!Number.isInteger(input.publicPort) || Number(input.publicPort) < 1 || Number(input.publicPort) > 65535)) throw new Error('Invalid public port.');
|
||||
return { hostId: host.id, libraryId: version.id, image, bindAddress, publicAddress: input.publicAddress as string | undefined ?? host.publicAddress,
|
||||
publicPort: input.publicPort as number | undefined };
|
||||
}
|
||||
|
||||
export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview {
|
||||
let recipe: NativeRecipe;
|
||||
let blueprintName: string | null = null;
|
||||
if (typeof input.deploymentId === 'string') {
|
||||
const previous = recipeForDeployment(input.deploymentId);
|
||||
if (!previous) throw new Error('The recorded deployment is unavailable.');
|
||||
recipe = captureRecipe(previous.recipe.game, previous.recipe.mapSlug);
|
||||
blueprintName = previous.blueprintName;
|
||||
} else if (typeof input.blueprintId === 'string') ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId));
|
||||
else {
|
||||
if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.');
|
||||
recipe = captureRecipe(input.game, input.mapSlug);
|
||||
}
|
||||
const selection = nativeSelection(input, recipe.game);
|
||||
recipe.native = selection; recipe.image = selection.image!;
|
||||
const base = `${recipe.game}_zombie_${recipe.mapSlug}`;
|
||||
const lobby = nextLobbyNumber(base);
|
||||
const name = lobby === null ? null : `${base}_${lobby}`;
|
||||
const port = nextNativePort(selection);
|
||||
const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
||||
db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
|
||||
const checks: LaunchPreview['checks'] = [
|
||||
{ label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` },
|
||||
{ label: 'Game files', state: 'pass', detail: `Immutable library version ${selection.libraryId} is ready on the selected host.` },
|
||||
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
|
||||
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
|
||||
{ label: 'Configuration', state: 'pass', detail: recipe.configSource }
|
||||
];
|
||||
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks,
|
||||
ready: checks.every(check => check.state !== 'blocked') };
|
||||
}
|
||||
|
||||
export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null {
|
||||
const excluded = [...taken];
|
||||
for (;;) {
|
||||
const port = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'udp');
|
||||
if (port === null) return null;
|
||||
// Both protocols are published by Plutainer and reserved together.
|
||||
const tcp = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'tcp');
|
||||
if (tcp === port) return port;
|
||||
if (tcp !== null && tcp < port) { excluded.push(tcp); continue; }
|
||||
excluded.push(port);
|
||||
}
|
||||
}
|
||||
|
||||
export function queueNativeCreation(input: CreateServerInput): CreateServerResult {
|
||||
try {
|
||||
return db().transaction(() => {
|
||||
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id=?').get(input.previewId) as
|
||||
{ id: string; recipe: string; server: string; port: number; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined;
|
||||
if (input.previewId && !preview) throw new Error('Review a fresh launch preview.');
|
||||
if (preview?.deployment_id) return { ok: true as const, name: preview.server, port: preview.port, deploymentId: preview.deployment_id };
|
||||
if (preview && preview.expires_at <= new Date().toISOString()) throw new Error('Launch preview expired.');
|
||||
const saved = preview ? JSON.parse(preview.recipe) as NativeRecipe : undefined;
|
||||
const game = input.game ?? 't5';
|
||||
if (input.gameMode !== 'zombie' || (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug))) throw new Error('Selection changed. Review a fresh launch preview.');
|
||||
const selection = nativeSelection((saved?.native ?? input) as unknown as Record<string, unknown>, game);
|
||||
const recipe: NativeRecipe = { ...captureRecipe(game, input.mapSlug), native: selection, image: selection.image! };
|
||||
const lobby = nextLobbyNumber(`${game}_zombie_${input.mapSlug}`);
|
||||
if (lobby === null) throw new Error('All lobby IDs are reserved.');
|
||||
const name = `${game}_zombie_${input.mapSlug}_${lobby}`;
|
||||
const port = nextNativePort(selection);
|
||||
if (!port) throw new Error('No available host port.');
|
||||
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || preview.port !== port))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
|
||||
const key = input.plutoKey?.trim() || getServerKey(name);
|
||||
if (!key) throw new Error(`Save a Plutonium server key for ${name}.`);
|
||||
saveServerKey(name, key);
|
||||
const instanceId = randomUUID();
|
||||
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'udp' });
|
||||
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'tcp' });
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, name, selection.hostId, selection.libraryId, selection.image, selection.bindAddress, port,
|
||||
selection.publicAddress ?? null, selection.publicPort ?? null, new Date().toISOString());
|
||||
const id = beginDeployment(name, 'create', input.createdBy);
|
||||
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
|
||||
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id);
|
||||
deploymentStage(id, 'Queued for native host deployment', { phase: 'startup-queue', recipeStatus: 'queued' });
|
||||
return { ok: true as const, name, port, deploymentId: id };
|
||||
})();
|
||||
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
|
||||
}
|
||||
|
||||
export function bindNativeOperation(id: string, server: string): void {
|
||||
const instance = nativeInstance(server);
|
||||
if (instance) db().prepare('INSERT OR IGNORE INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,?)').run(id, instance.id, instance.generation);
|
||||
}
|
||||
|
||||
/** Cancelling an unsubmitted launch releases only its local reservation. */
|
||||
export function cancelQueuedNativeCreation(server: string, triggeredBy: string): string | null {
|
||||
const instance = nativeInstance(server);
|
||||
if (!instance || getServer(server)) return null;
|
||||
return db().transaction(() => {
|
||||
const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined;
|
||||
if (!creation) return null;
|
||||
const id = beginDeployment(server, 'destroy', triggeredBy);
|
||||
const now = new Date().toISOString();
|
||||
db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage='Queued native creation cancelled',server_deleted_at=? WHERE id=?")
|
||||
.run(now, now, creation.id);
|
||||
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(now, instance.id);
|
||||
releaseHostPorts(instance.id);
|
||||
deploymentStage(id, 'Cancelled queued creation; no agent mutation was submitted');
|
||||
finishDeployment(id, { ok: true });
|
||||
return id;
|
||||
})();
|
||||
}
|
||||
|
||||
export function queueNativeRebuild(name: string, triggeredBy: string): CreateServerResult {
|
||||
try {
|
||||
const instance = nativeInstance(name)!; const server = getServer(name);
|
||||
const blocker = activeBetaRemovalBlocker(name);
|
||||
if (blocker) throw new Error(blocker);
|
||||
if (!server?.map_slug || !getServerKey(name)) throw new Error('A saved map and server key are required before rebuilding.');
|
||||
requireReadyLibrary(instance.host_id, instance.library_id);
|
||||
const recipe: NativeRecipe = { ...captureRecipe(server.game as 't5' | 't6', server.map_slug), image: instance.image,
|
||||
native: { hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address } };
|
||||
return db().transaction(() => {
|
||||
const id = beginDeployment(name, 'rebuild', triggeredBy);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)').run(id, JSON.stringify(recipe));
|
||||
deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' });
|
||||
return { ok: true as const, name, port: instance.port, deploymentId: id };
|
||||
})();
|
||||
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
|
||||
}
|
||||
|
||||
function filesFor(job: Job): FilePlan[] {
|
||||
const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s
|
||||
ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
|
||||
WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as StagedFileChange[];
|
||||
return rows.map(row => ({ id: row.id, path: posix.join(relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), row.rel_path),
|
||||
contentBase64: row.content?.toString('base64') ?? null, sha256: row.content ? sha256Buffer(row.content) : null,
|
||||
checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha }));
|
||||
}
|
||||
|
||||
/** Agent receipts are the authoritative before images; do not read a controller mount. */
|
||||
function applyReceipt(job: Job, result: Record<string, unknown>) {
|
||||
if (job.applied_at || !job.files_json) return;
|
||||
const plans = JSON.parse(job.files_json) as FilePlan[];
|
||||
const before = result.before as { id?: number; path: string; contentBase64: string | null }[] | undefined;
|
||||
db().transaction(() => {
|
||||
for (const plan of plans) {
|
||||
const snapshot = before?.find(item => item.id === plan.id || item.path === plan.path);
|
||||
if (!snapshot) throw new Error('Agent activation receipt is missing a before image.');
|
||||
const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange;
|
||||
if (!row || row.state === 'applied') continue;
|
||||
const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64');
|
||||
const prior = getServerScript(job.server, row.rel_path);
|
||||
if (row.source_kind === 'beta') {
|
||||
if (bytes !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots(beta_id,server,rel_path,prior_content,prior_sha256,prior_size) VALUES(?,?,?,?,?,?)`)
|
||||
.run(row.source_id, job.server, row.rel_path, bytes, sha256Buffer(bytes), bytes.length);
|
||||
db().prepare(`UPDATE beta_server_scripts SET had_original=?,prior_version_id=? WHERE beta_id=? AND server=?
|
||||
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path=? AND kind=?)`).run(bytes === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, job.server, row.rel_path, row.kind);
|
||||
db().prepare('INSERT OR IGNORE INTO beta_file_changes(beta_id,server,rel_path) VALUES(?,?,?)').run(row.source_id, job.server, row.rel_path);
|
||||
db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server);
|
||||
}
|
||||
if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?')
|
||||
.run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
|
||||
if (row.content === null) removeScript(job.server, row.rel_path);
|
||||
else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null,
|
||||
sha256Buffer(row.content), row.content.length, row.inventory_source);
|
||||
db().prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?").run(new Date().toISOString(), row.id);
|
||||
}
|
||||
db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
|
||||
})();
|
||||
}
|
||||
|
||||
function createPayload(instance: NativeInstance, recipe: LaunchRecipe) {
|
||||
const key = getServerKey(instance.server);
|
||||
if (!key) throw new Error('The saved server key was removed.');
|
||||
return { name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, port: instance.port, bindAddress: instance.bind_address,
|
||||
rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
|
||||
env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp',
|
||||
PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: key },
|
||||
files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), contentBase64: file.content, sha256: file.sha256 })) };
|
||||
}
|
||||
|
||||
function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
|
||||
function complete(job: Job) {
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
|
||||
db().prepare('UPDATE restart_rollout SET healthy_at=?,error=NULL WHERE operation_id=?').run(new Date().toISOString(), job.id);
|
||||
finishDeployment(job.id, { ok: true });
|
||||
})();
|
||||
}
|
||||
|
||||
export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null): Promise<void> {
|
||||
// Draining prevents admission immediately and retires existing lobbies only through the normal safety queue.
|
||||
for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) {
|
||||
const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
|
||||
for (const instance of instances) {
|
||||
if (!getServer(instance.server) || activeBetaRemovalBlocker(instance.server)) continue;
|
||||
if (db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')").get(instance.server)) continue;
|
||||
const id = beginDeployment(instance.server, 'destroy', 'host-drain');
|
||||
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
|
||||
deploymentStage(id, 'Host draining; waiting for a fresh empty lobby');
|
||||
}
|
||||
}
|
||||
const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at
|
||||
FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
||||
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
|
||||
for (const job of jobs) {
|
||||
try { await processJob(job, eligibility); }
|
||||
catch (error) { hold(job, (error as Error).message); }
|
||||
}
|
||||
}
|
||||
|
||||
async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
let instance = nativeInstance(job.server);
|
||||
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
|
||||
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
|
||||
}
|
||||
const host = getHost(instance.host_id);
|
||||
if (!host?.connected || host.mode === 'disabled') {
|
||||
db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
|
||||
hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return;
|
||||
}
|
||||
if (job.operation_id) {
|
||||
const operation = getHostOperation(job.operation_id);
|
||||
if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
|
||||
if (operation.state === 'failed') {
|
||||
// The agent reports failed only when no mutation began. Ambiguous
|
||||
// post-mutation failures remain unknown and retain their capacity.
|
||||
db().transaction(() => {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id);
|
||||
const files = JSON.parse(job.files_json ?? '[]') as FilePlan[];
|
||||
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='pending' WHERE id=? AND state='applying'").run(file.id);
|
||||
finishDeployment(job.id, { ok: false, message: operation.error ?? 'Agent operation failed before mutation.' });
|
||||
})();
|
||||
return;
|
||||
}
|
||||
if (operation.state !== 'completed') {
|
||||
const blocked = operation.result as { message?: string } | null;
|
||||
hold(job, blocked?.message ?? `Agent operation ${operation.state}`); return;
|
||||
}
|
||||
if (job.action === 'destroy') {
|
||||
db().transaction(() => {
|
||||
deploymentStage(job.id, REMOVING_SERVER, { checkpoint: { key: 'delete-deployment', label: 'Agent confirmed instance and container removal' } });
|
||||
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance!.id);
|
||||
releaseHostPorts(instance!.id);
|
||||
db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
|
||||
complete(job);
|
||||
})();
|
||||
return;
|
||||
}
|
||||
if (job.action === 'rebuild' && instance.generation === job.generation) {
|
||||
db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation);
|
||||
instance = nativeInstance(job.server)!;
|
||||
}
|
||||
applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
|
||||
if (job.state !== 'awaiting-health') {
|
||||
db().prepare("UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?").run(job.id);
|
||||
recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` }, phase: 'verify-container' });
|
||||
}
|
||||
const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect');
|
||||
if (job.action === 'stop') {
|
||||
if (!['stopped','exited','created','dead'].includes(health.state)) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; }
|
||||
db().prepare("UPDATE servers SET container_state=?,last_players=NULL,last_roster=NULL WHERE name=?").run(health.state, job.server);
|
||||
complete(job); return;
|
||||
}
|
||||
if (!health.healthy) { hold(job, `Waiting for container readiness: ${health.state}`); return; }
|
||||
const status = await probeServerStatus(job.server, operation.updatedAt);
|
||||
if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; }
|
||||
if (job.action === 'create' || job.action === 'rebuild') {
|
||||
const recipe = recipeForDeployment(job.id)!.recipe;
|
||||
if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
|
||||
await checkRelease();
|
||||
const revision = await probeVersion(job.server); const target = freshRelease();
|
||||
if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
|
||||
if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
|
||||
}
|
||||
db().prepare("UPDATE servers SET container_state='running',container_checked_at=? WHERE name=?").run(new Date().toISOString(), job.server);
|
||||
recordProgress(job.id, { blocker: null, checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', actual: health.state },
|
||||
{ key: 'game', label: 'Game status', source: status.source, result: 'pass', actual: status.players }] });
|
||||
complete(job); return;
|
||||
}
|
||||
const reason = eligibility(job);
|
||||
if (reason) { hold(job, reason); return; }
|
||||
if (job.action === 'destroy' || job.action === 'rebuild') {
|
||||
const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; }
|
||||
}
|
||||
const files = job.files_json ? JSON.parse(job.files_json) as FilePlan[] : filesFor(job);
|
||||
if (files.length && !job.stage_operation_id) {
|
||||
const stage = queueHostOperation(instance.host_id, 'files.stage', { stageId: job.id, files }, { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:stage` });
|
||||
db().prepare('UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?').run(stage.id, JSON.stringify(files), job.id);
|
||||
hold(job, 'Staging and verifying files on the host.'); return;
|
||||
}
|
||||
if (job.stage_operation_id) {
|
||||
const stage = getHostOperation(job.stage_operation_id);
|
||||
if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; }
|
||||
if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; }
|
||||
if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) {
|
||||
finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return;
|
||||
}
|
||||
}
|
||||
const settings = fleetSettings();
|
||||
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
|
||||
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
|
||||
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
|
||||
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
|
||||
if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 });
|
||||
if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
|
||||
if (job.action !== 'create' && job.action !== 'start') {
|
||||
// Fast controller check keeps occupied lobbies out of the operation slots.
|
||||
// The agent repeats this immediately before its local mutation.
|
||||
const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
|
||||
if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
|
||||
const occupancy = await probePlayerEvidence(job.server);
|
||||
const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s
|
||||
JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
|
||||
WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
|
||||
if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return; }
|
||||
if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return; }
|
||||
}
|
||||
}
|
||||
const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
|
||||
if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
|
||||
db().transaction(() => {
|
||||
const live = nativeInstance(job.server);
|
||||
if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
|
||||
if (job.action === 'destroy' || job.action === 'rebuild') {
|
||||
const beta = activeBetaRemovalBlocker(job.server);
|
||||
if (beta) { hold(job, beta); return; }
|
||||
}
|
||||
if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
|
||||
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
|
||||
}
|
||||
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
|
||||
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
|
||||
DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
|
||||
}
|
||||
let payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
|
||||
if (job.action === 'create' || job.action === 'rebuild') {
|
||||
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
|
||||
const recipe: NativeRecipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image };
|
||||
payload = createPayload(instance!, recipe);
|
||||
if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1;
|
||||
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
|
||||
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
|
||||
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
|
||||
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
|
||||
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
|
||||
}
|
||||
const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload,
|
||||
{ instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id });
|
||||
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
|
||||
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
|
||||
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
|
||||
deploymentStage(job.id, `Host agent ${job.action} queued`, { phase: 'container-action', blocker: null,
|
||||
command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
|
||||
})();
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { queueRebuildServer } from './instances.js';
|
||||
import { restartRequestInFlight } from './restarts.js';
|
||||
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { nativeInstance } from '../backends/native.js';
|
||||
|
||||
export interface LifecycleOperation {
|
||||
id: string; server: string; action: string; state: string; stage: string;
|
||||
@@ -56,9 +57,10 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
|
||||
// stale file preconditions after a beta; its failure must not strand
|
||||
// the rollback that would restore the server.
|
||||
const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id);
|
||||
if (source.kind === 'campaign' && !row.restarted_at && !rollback) {
|
||||
if (source.kind === 'campaign' && !row.restarted_at && !rollback && !nativeInstance(server)) {
|
||||
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart'
|
||||
AND d.state = 'waiting' AND d.submitted_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.stage_operation_id IS NOT NULL)
|
||||
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
|
||||
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
|
||||
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id
|
||||
@@ -135,7 +137,7 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo
|
||||
|
||||
export function requestLifecycle(server: string, action: RuntimeAction) {
|
||||
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
|
||||
if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
|
||||
if (!nativeInstance(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' };
|
||||
return db().transaction(() => {
|
||||
const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
|
||||
if (blocker) return { ok: false as const, status: 409, error: blocker };
|
||||
@@ -184,7 +186,7 @@ function watchdogRequestGuard(server: string, incidentId: string): Extract<Lifec
|
||||
const row = getServer(server);
|
||||
if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' };
|
||||
if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' };
|
||||
if (!komodoConfigured()) return { ok: false, status: 503, error: 'Configure Komodo before provisioning.' };
|
||||
if (!nativeInstance(server) && !komodoConfigured()) return { ok: false, status: 503, error: 'Configure a host before provisioning.' };
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
|
||||
import type { RuntimeAction } from '../../provisioning.js';
|
||||
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
||||
import { recordProgress, type ProgressUpdate } from './progress.js';
|
||||
import { nativeDeployment } from '../backends/native.js';
|
||||
|
||||
// Komodo queues restart work. Don't mistake the still-running old container
|
||||
// for a completed restart when an API kick immediately follows the request.
|
||||
@@ -166,6 +167,7 @@ async function refresh(): Promise<void> {
|
||||
const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
|
||||
.all() as RestartSlot[];
|
||||
for (const slot of slots) {
|
||||
if (slot.operation_id && nativeDeployment(slot.operation_id)) continue;
|
||||
if (requestsInFlight.has(slot.server)) continue;
|
||||
// Recovery capacity transferred to a rebuild remains reserved until
|
||||
// teardown. Its health is verified by the creation worker, never here.
|
||||
|
||||
@@ -12,6 +12,8 @@ import { cancelProvisioning } from './requests.js';
|
||||
import type { RuntimeAction } from '../../provisioning.js';
|
||||
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { nativeInstance, nativeDeployment } from '../backends/native.js';
|
||||
import { processNativeProvisioning } from './native.js';
|
||||
|
||||
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
|
||||
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
|
||||
@@ -98,6 +100,19 @@ export function processProvisioning(): Promise<void> {
|
||||
export const processLifecycle = processProvisioning;
|
||||
|
||||
async function run(): Promise<void> {
|
||||
await processNativeProvisioning(operation => {
|
||||
if (operation.action === 'create') return null;
|
||||
if (operation.action === 'destroy') {
|
||||
const other = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL
|
||||
AND (state='running' OR (state='awaiting-health' AND action NOT IN ('create','rebuild'))) LIMIT 1`).get(operation.server, operation.id);
|
||||
return other ? 'Waiting for the current server operation.' : null;
|
||||
}
|
||||
const typed = operation as Operation;
|
||||
const eligibility = sourceEligibility(typed);
|
||||
if (eligibility === 'cancelled') { cancelProvisioning(operation.id, 'Requester cancelled or replaced this operation'); return 'Requester cancelled this operation.'; }
|
||||
if (eligibility === 'held') return 'Waiting for the requester to resume.';
|
||||
return blocker(operation.server, operation.id, typed.action)?.message ?? null;
|
||||
});
|
||||
await processCreations();
|
||||
await processDestructions();
|
||||
await refreshRestartRollout();
|
||||
@@ -105,6 +120,7 @@ async function run(): Promise<void> {
|
||||
WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
|
||||
ORDER BY started_at, rowid`).all() as Operation[];
|
||||
for (const operation of operations) {
|
||||
if (nativeDeployment(operation.id)) continue;
|
||||
try { await processOperation(operation); }
|
||||
catch (err) {
|
||||
// Observation failures are retried; remote mutations are never replayed.
|
||||
|
||||
Reference in New Issue
Block a user