Add Prettier and ESLint tooling (Refs #77)
This commit is contained in:
@@ -29,41 +29,81 @@ const scheduled = new WeakSet<ReturnType<typeof db>>();
|
||||
function snapshotSchemaReady(handle: ReturnType<typeof db>): boolean {
|
||||
// Host enrollment and game-file maintenance can run before provisioning is
|
||||
// initialized. Missing queue data means unknown capacity, not an empty fleet.
|
||||
const { count } = handle.prepare(`SELECT COUNT(*) AS count FROM sqlite_master WHERE type='table'
|
||||
AND name IN ('native_instances','native_provisioning','server_deployments','settings')`).get() as { count: number };
|
||||
const { count } = handle
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count FROM sqlite_master WHERE type='table'
|
||||
AND name IN ('native_instances','native_provisioning','server_deployments','settings')`
|
||||
)
|
||||
.get() as { count: number };
|
||||
if (count === 4) return true;
|
||||
snapshots.delete(handle);
|
||||
return false;
|
||||
}
|
||||
|
||||
function queueSnapshot(handle: ReturnType<typeof db>, hosts: Map<string, HostAvailability>): Snapshot {
|
||||
const counts = handle.prepare(`SELECT
|
||||
function queueSnapshot(
|
||||
handle: ReturnType<typeof db>,
|
||||
hosts: Map<string, HostAvailability>
|
||||
): Snapshot {
|
||||
const counts = handle
|
||||
.prepare(
|
||||
`SELECT
|
||||
COALESCE(SUM(state IN ('running','awaiting-health') OR ${committedRebalanceStartup}),0) AS active,
|
||||
COALESCE(SUM(state='queued'),0) AS queued FROM server_deployments
|
||||
WHERE action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup})
|
||||
AND server_deleted_at IS NULL`).get() as { active: number; queued: number };
|
||||
return { hosts, ...counts, startupLimit: fleetSettings().createConcurrency, updatedAt: new Date().toISOString() };
|
||||
AND server_deleted_at IS NULL`
|
||||
)
|
||||
.get() as { active: number; queued: number };
|
||||
return {
|
||||
hosts,
|
||||
...counts,
|
||||
startupLimit: fleetSettings().createConcurrency,
|
||||
updatedAt: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
function observeHost(host: Host): HostAvailability {
|
||||
const handle = db();
|
||||
const { running, startup } = capacityCalibration(host.id);
|
||||
const measured = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample.generation]));
|
||||
const pending = handle.prepare(`SELECT DISTINCT i.id,i.generation FROM native_instances i
|
||||
const measured = new Map(
|
||||
host.inventory.instanceResources?.map((sample) => [sample.instanceId, sample.generation])
|
||||
);
|
||||
const pending = handle
|
||||
.prepare(
|
||||
`SELECT DISTINCT i.id,i.generation FROM native_instances i
|
||||
JOIN native_provisioning n ON n.instance_id=i.id JOIN server_deployments d ON d.id=n.deployment_id
|
||||
WHERE i.host_id=? AND i.retired_at IS NULL AND d.action IN ('create','rebuild')
|
||||
AND d.state IN ('running','awaiting-health')`).all(host.id) as { id: string; generation: number }[];
|
||||
const incoming = handle.prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
|
||||
WHERE m.target_host_id=? AND (m.removed_at IS NULL OR d.state IN ('queued','waiting','interrupted'))`).get(host.id) as { n: number };
|
||||
const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length + incoming.n;
|
||||
AND d.state IN ('running','awaiting-health')`
|
||||
)
|
||||
.all(host.id) as { id: string; generation: number }[];
|
||||
const incoming = handle
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS n FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
|
||||
WHERE m.target_host_id=? AND (m.removed_at IS NULL OR d.state IN ('queued','waiting','interrupted'))`
|
||||
)
|
||||
.get(host.id) as { n: number };
|
||||
const unmeasured =
|
||||
pending.filter((instance) => measured.get(instance.id) !== instance.generation).length +
|
||||
incoming.n;
|
||||
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
|
||||
const freeCpu = cpuCount && cpuUsagePercent !== undefined
|
||||
? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores) : null;
|
||||
const freeMemory = memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal
|
||||
? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes) : null;
|
||||
return { host, freeCpu, freeMemory, runningCpu: running.cpuCores, runningMemory: running.memoryBytes,
|
||||
const freeCpu =
|
||||
cpuCount && cpuUsagePercent !== undefined
|
||||
? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores)
|
||||
: null;
|
||||
const freeMemory =
|
||||
memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal
|
||||
? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes)
|
||||
: null;
|
||||
return {
|
||||
host,
|
||||
freeCpu,
|
||||
freeMemory,
|
||||
runningCpu: running.cpuCores,
|
||||
runningMemory: running.memoryBytes,
|
||||
calibration: { running, startup },
|
||||
ports: handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as HostAvailability['ports'] };
|
||||
ports: handle
|
||||
.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?')
|
||||
.all(host.id) as HostAvailability['ports']
|
||||
};
|
||||
}
|
||||
|
||||
/** Host telemetry updates the estimate once, independently of how many lobbies are previewed. */
|
||||
@@ -85,20 +125,33 @@ export function scheduleAvailabilityRefresh(): void {
|
||||
if (!handle.open) return;
|
||||
try {
|
||||
if (!snapshotSchemaReady(handle)) return;
|
||||
const hosts = new Map(listHosts().map(host => [host.id, observeHost(host)]));
|
||||
const hosts = new Map(listHosts().map((host) => [host.id, observeHost(host)]));
|
||||
snapshots.set(handle, queueSnapshot(handle, hosts));
|
||||
} catch (error) { console.error('[availability] snapshot refresh failed:', error); }
|
||||
} catch (error) {
|
||||
console.error('[availability] snapshot refresh failed:', error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
subscribe(event => {
|
||||
if (event.type === 'fleet-invalidated' || event.type === 'server-status') scheduleAvailabilityRefresh();
|
||||
subscribe((event) => {
|
||||
if (event.type === 'fleet-invalidated' || event.type === 'server-status')
|
||||
scheduleAvailabilityRefresh();
|
||||
});
|
||||
|
||||
export function freePorts(snapshot: HostAvailability, bindAddress?: string): number {
|
||||
const bind = bindAddress ?? snapshot.host.bindAddress;
|
||||
const used = new Set(snapshot.ports.filter(row => row.port >= snapshot.host.portMin && row.port <= snapshot.host.portMax
|
||||
&& (row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(row.bind_address))).map(row => row.port));
|
||||
const used = new Set(
|
||||
snapshot.ports
|
||||
.filter(
|
||||
(row) =>
|
||||
row.port >= snapshot.host.portMin &&
|
||||
row.port <= snapshot.host.portMax &&
|
||||
(row.bind_address === bind ||
|
||||
['0.0.0.0', '::'].includes(bind) ||
|
||||
['0.0.0.0', '::'].includes(row.bind_address))
|
||||
)
|
||||
.map((row) => row.port)
|
||||
);
|
||||
return Math.max(0, snapshot.host.portMax - snapshot.host.portMin + 1 - used.size);
|
||||
}
|
||||
|
||||
@@ -110,44 +163,108 @@ export function readAvailabilitySnapshot(): Snapshot | null {
|
||||
}
|
||||
|
||||
/** Estimates only: no placement, historical scans, reservations, or live host requests. */
|
||||
export function launchAvailability(selections: NativePlacement[]): { availability: LaunchAvailability; warnings: string[] } {
|
||||
export function launchAvailability(selections: NativePlacement[]): {
|
||||
availability: LaunchAvailability;
|
||||
warnings: string[];
|
||||
} {
|
||||
const snapshot = snapshots.get(db());
|
||||
if (!snapshot) {
|
||||
scheduleAvailabilityRefresh();
|
||||
return { availability: { updatedAt: null, requested: selections.length, estimatedAvailable: null, portsAvailable: null,
|
||||
cpuCoresAvailable: null, memoryBytesAvailable: null, startupSlots: null, startupLimit: null, queued: null },
|
||||
warnings: selections.length ? ['Capacity measurements are updating. You can queue this launch; provisioning waits for an eligible host.'] : [] };
|
||||
return {
|
||||
availability: {
|
||||
updatedAt: null,
|
||||
requested: selections.length,
|
||||
estimatedAvailable: null,
|
||||
portsAvailable: null,
|
||||
cpuCoresAvailable: null,
|
||||
memoryBytesAvailable: null,
|
||||
startupSlots: null,
|
||||
startupLimit: null,
|
||||
queued: null
|
||||
},
|
||||
warnings: selections.length
|
||||
? [
|
||||
'Capacity measurements are updating. You can queue this launch; provisioning waits for an eligible host.'
|
||||
]
|
||||
: []
|
||||
};
|
||||
}
|
||||
const candidates = [...snapshot.hosts.values()].filter(({ host }) => host.readiness === 'ready'
|
||||
&& host.lastHeartbeat && Date.now() - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS
|
||||
&& (host.inventory.storageAvailable ?? 0) >= host.minimumFreeBytes + 1024 ** 3);
|
||||
const eligible = (candidate: HostAvailability, selection: NativePlacement) => (!selection.hostId || selection.automatic || candidate.host.id === selection.hostId)
|
||||
&& !!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '')
|
||||
&& !!candidate.host.inventory.libraries?.some(library => library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state));
|
||||
const relevant = candidates.filter(candidate => selections.some(selection => eligible(candidate, selection)));
|
||||
const candidates = [...snapshot.hosts.values()].filter(
|
||||
({ host }) =>
|
||||
host.readiness === 'ready' &&
|
||||
host.lastHeartbeat &&
|
||||
Date.now() - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS &&
|
||||
(host.inventory.storageAvailable ?? 0) >= host.minimumFreeBytes + 1024 ** 3
|
||||
);
|
||||
const eligible = (candidate: HostAvailability, selection: NativePlacement) =>
|
||||
(!selection.hostId || selection.automatic || candidate.host.id === selection.hostId) &&
|
||||
!!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '') &&
|
||||
!!candidate.host.inventory.libraries?.some(
|
||||
(library) =>
|
||||
library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state)
|
||||
);
|
||||
const relevant = candidates.filter((candidate) =>
|
||||
selections.some((selection) => eligible(candidate, selection))
|
||||
);
|
||||
const planned = new Map<string, number>();
|
||||
let estimatedAvailable = 0;
|
||||
for (const selection of selections) {
|
||||
const candidate = relevant.find(candidate => {
|
||||
const candidate = relevant.find((candidate) => {
|
||||
const used = planned.get(candidate.host.id) ?? 0;
|
||||
return eligible(candidate, selection) && freePorts(candidate, selection.network?.bindAddress ?? selection.bindAddress) > used
|
||||
&& candidate.freeCpu !== null && candidate.freeMemory !== null
|
||||
&& candidate.freeCpu >= (used + 1) * candidate.runningCpu
|
||||
&& candidate.freeMemory >= (used + 1) * candidate.runningMemory;
|
||||
return (
|
||||
eligible(candidate, selection) &&
|
||||
freePorts(candidate, selection.network?.bindAddress ?? selection.bindAddress) >
|
||||
used &&
|
||||
candidate.freeCpu !== null &&
|
||||
candidate.freeMemory !== null &&
|
||||
candidate.freeCpu >= (used + 1) * candidate.runningCpu &&
|
||||
candidate.freeMemory >= (used + 1) * candidate.runningMemory
|
||||
);
|
||||
});
|
||||
if (candidate) { estimatedAvailable++; planned.set(candidate.host.id, (planned.get(candidate.host.id) ?? 0) + 1); }
|
||||
if (candidate) {
|
||||
estimatedAvailable++;
|
||||
planned.set(candidate.host.id, (planned.get(candidate.host.id) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
const warnings: string[] = [];
|
||||
const unknown = relevant.some(candidate => candidate.freeCpu === null || candidate.freeMemory === null);
|
||||
if (selections.length && !relevant.length) warnings.push('Waiting for an eligible host with the selected runtime and game files. The launch can be queued now.');
|
||||
else if (unknown) warnings.push('Some host CPU or memory measurements are unavailable. Provisioning waits for fresh capacity measurements.');
|
||||
else if (estimatedAvailable < selections.length) warnings.push(`Current capacity is estimated for ${estimatedAvailable} of ${selections.length} servers. Remaining launches wait for capacity, including any removals in this collection.`);
|
||||
const unknown = relevant.some(
|
||||
(candidate) => candidate.freeCpu === null || candidate.freeMemory === null
|
||||
);
|
||||
if (selections.length && !relevant.length)
|
||||
warnings.push(
|
||||
'Waiting for an eligible host with the selected runtime and game files. The launch can be queued now.'
|
||||
);
|
||||
else if (unknown)
|
||||
warnings.push(
|
||||
'Some host CPU or memory measurements are unavailable. Provisioning waits for fresh capacity measurements.'
|
||||
);
|
||||
else if (estimatedAvailable < selections.length)
|
||||
warnings.push(
|
||||
`Current capacity is estimated for ${estimatedAvailable} of ${selections.length} servers. Remaining launches wait for capacity, including any removals in this collection.`
|
||||
);
|
||||
const startupSlots = Math.max(0, snapshot.startupLimit - snapshot.active);
|
||||
if (selections.length && !startupSlots) warnings.push('All startup slots are in use. New launches will wait in the queue.');
|
||||
if (selections.length && snapshot.queued) warnings.push(`${snapshot.queued} server${snapshot.queued === 1 ? ' is' : 's are'} already queued.`);
|
||||
return { availability: { updatedAt: snapshot.updatedAt, requested: selections.length, estimatedAvailable: unknown ? null : estimatedAvailable,
|
||||
portsAvailable: relevant.reduce((total, candidate) => total + freePorts(candidate), 0),
|
||||
cpuCoresAvailable: unknown ? null : relevant.reduce((total, candidate) => total + (candidate.freeCpu ?? 0), 0),
|
||||
memoryBytesAvailable: unknown ? null : relevant.reduce((total, candidate) => total + (candidate.freeMemory ?? 0), 0),
|
||||
startupSlots, startupLimit: snapshot.startupLimit, queued: snapshot.queued }, warnings };
|
||||
if (selections.length && !startupSlots)
|
||||
warnings.push('All startup slots are in use. New launches will wait in the queue.');
|
||||
if (selections.length && snapshot.queued)
|
||||
warnings.push(
|
||||
`${snapshot.queued} server${snapshot.queued === 1 ? ' is' : 's are'} already queued.`
|
||||
);
|
||||
return {
|
||||
availability: {
|
||||
updatedAt: snapshot.updatedAt,
|
||||
requested: selections.length,
|
||||
estimatedAvailable: unknown ? null : estimatedAvailable,
|
||||
portsAvailable: relevant.reduce((total, candidate) => total + freePorts(candidate), 0),
|
||||
cpuCoresAvailable: unknown
|
||||
? null
|
||||
: relevant.reduce((total, candidate) => total + (candidate.freeCpu ?? 0), 0),
|
||||
memoryBytesAvailable: unknown
|
||||
? null
|
||||
: relevant.reduce((total, candidate) => total + (candidate.freeMemory ?? 0), 0),
|
||||
startupSlots,
|
||||
startupLimit: snapshot.startupLimit,
|
||||
queued: snapshot.queued
|
||||
},
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureHostSchema } from './schema.js';
|
||||
import type { CapacityCalibration, CapacitySample, CapacitySamplePage, HostInventory, ResourceAllowance } from '../../hosts.js';
|
||||
import type {
|
||||
CapacityCalibration,
|
||||
CapacitySample,
|
||||
CapacitySamplePage,
|
||||
HostInventory,
|
||||
ResourceAllowance
|
||||
} from '../../hosts.js';
|
||||
|
||||
const BOOTSTRAP_CPU = 0.5;
|
||||
const BOOTSTRAP_MEMORY = 1024 ** 3;
|
||||
@@ -9,59 +15,114 @@ const WINDOW_MS = 24 * 60 * 60_000;
|
||||
const MAX_SAMPLES = 10_000;
|
||||
type Phase = 'startup' | 'running';
|
||||
interface AllowanceRow {
|
||||
scope: string; phase: Phase; cpu_cores: number; memory_bytes: number; samples: number;
|
||||
window_start: string; observed_at: string; computed_at: string;
|
||||
scope: string;
|
||||
phase: Phase;
|
||||
cpu_cores: number;
|
||||
memory_bytes: number;
|
||||
samples: number;
|
||||
window_start: string;
|
||||
observed_at: string;
|
||||
computed_at: string;
|
||||
}
|
||||
function database() {
|
||||
const handle = db();
|
||||
ensureHostSchema(handle);
|
||||
return handle;
|
||||
}
|
||||
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
|
||||
|
||||
/** Read retained minute peaks without contacting agents or recalculating allowances. */
|
||||
export function capacitySamples(hostId: string, options: { scope: 'host' | 'fleet'; phase: 'all' | Phase; page: number }): CapacitySamplePage {
|
||||
export function capacitySamples(
|
||||
hostId: string,
|
||||
options: { scope: 'host' | 'fleet'; phase: 'all' | Phase; page: number }
|
||||
): CapacitySamplePage {
|
||||
const handle = database();
|
||||
const clauses = ['s.bucket >= ?'];
|
||||
const values: (string | number)[] = [new Date(Date.now() - WINDOW_MS).toISOString()];
|
||||
if (options.scope === 'host') { clauses.push('s.host_id = ?'); values.push(hostId); }
|
||||
if (options.phase !== 'all') { clauses.push('s.phase = ?'); values.push(options.phase); }
|
||||
if (options.scope === 'host') {
|
||||
clauses.push('s.host_id = ?');
|
||||
values.push(hostId);
|
||||
}
|
||||
if (options.phase !== 'all') {
|
||||
clauses.push('s.phase = ?');
|
||||
values.push(options.phase);
|
||||
}
|
||||
const where = clauses.join(' AND ');
|
||||
const total = (handle.prepare(`SELECT COUNT(*) AS count FROM ec_instance_resource_samples s WHERE ${where}`).get(...values) as { count: number }).count;
|
||||
const total = (
|
||||
handle
|
||||
.prepare(`SELECT COUNT(*) AS count FROM ec_instance_resource_samples s WHERE ${where}`)
|
||||
.get(...values) as { count: number }
|
||||
).count;
|
||||
const pageSize = 50;
|
||||
const page = Math.min(options.page, Math.max(1, Math.ceil(total / pageSize)));
|
||||
const samples = handle.prepare(`SELECT s.host_id AS hostId, h.name AS hostName, s.instance_id AS instanceId,
|
||||
const samples = handle
|
||||
.prepare(
|
||||
`SELECT s.host_id AS hostId, h.name AS hostName, s.instance_id AS instanceId,
|
||||
i.server, s.generation, s.phase, s.bucket AS timestamp, s.cpu_cores AS cpuCores, s.memory_bytes AS memoryBytes
|
||||
FROM ec_instance_resource_samples s
|
||||
LEFT JOIN ec_hosts h ON h.id=s.host_id
|
||||
LEFT JOIN native_instances i ON i.id=s.instance_id AND i.host_id=s.host_id
|
||||
WHERE ${where} ORDER BY s.bucket DESC, s.host_id, s.instance_id, s.generation, s.phase LIMIT ? OFFSET ?`)
|
||||
WHERE ${where} ORDER BY s.bucket DESC, s.host_id, s.instance_id, s.generation, s.phase LIMIT ? OFFSET ?`
|
||||
)
|
||||
.all(...values, pageSize, (page - 1) * pageSize) as CapacitySample[];
|
||||
return { samples, total, page, pageSize };
|
||||
}
|
||||
|
||||
export function validateInstanceResources(inventory: HostInventory): void {
|
||||
if (inventory.instanceResourcesError !== undefined && (typeof inventory.instanceResourcesError !== 'string'
|
||||
|| inventory.instanceResourcesError.length > 500)) throw new Error('Invalid instance resource sampling error.');
|
||||
if (
|
||||
inventory.instanceResourcesError !== undefined &&
|
||||
(typeof inventory.instanceResourcesError !== 'string' ||
|
||||
inventory.instanceResourcesError.length > 500)
|
||||
)
|
||||
throw new Error('Invalid instance resource sampling error.');
|
||||
const samples = inventory.instanceResources;
|
||||
if (samples === undefined) return;
|
||||
if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.');
|
||||
if (!Array.isArray(samples) || samples.length > 1000)
|
||||
throw new Error('Invalid instance resource samples.');
|
||||
const seen = new Set<string>();
|
||||
for (const sample of samples) {
|
||||
if (!sample || typeof sample !== 'object' || typeof sample.instanceId !== 'string' || !sample.instanceId || sample.instanceId.length > 100
|
||||
|| !Number.isSafeInteger(sample.generation) || sample.generation < 1
|
||||
|| !Number.isSafeInteger(sample.uptimeSeconds) || sample.uptimeSeconds < 0
|
||||
|| typeof sample.cpuCores !== 'number' || !Number.isFinite(sample.cpuCores) || sample.cpuCores < 0
|
||||
|| !Number.isSafeInteger(sample.memoryBytes) || sample.memoryBytes <= 0
|
||||
|| (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01)
|
||||
|| (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal)
|
||||
|| seen.has(sample.instanceId)) throw new Error('Invalid instance resource sample.');
|
||||
if (
|
||||
!sample ||
|
||||
typeof sample !== 'object' ||
|
||||
typeof sample.instanceId !== 'string' ||
|
||||
!sample.instanceId ||
|
||||
sample.instanceId.length > 100 ||
|
||||
!Number.isSafeInteger(sample.generation) ||
|
||||
sample.generation < 1 ||
|
||||
!Number.isSafeInteger(sample.uptimeSeconds) ||
|
||||
sample.uptimeSeconds < 0 ||
|
||||
typeof sample.cpuCores !== 'number' ||
|
||||
!Number.isFinite(sample.cpuCores) ||
|
||||
sample.cpuCores < 0 ||
|
||||
!Number.isSafeInteger(sample.memoryBytes) ||
|
||||
sample.memoryBytes <= 0 ||
|
||||
(inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01) ||
|
||||
(inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal) ||
|
||||
seen.has(sample.instanceId)
|
||||
)
|
||||
throw new Error('Invalid instance resource sample.');
|
||||
seen.add(sample.instanceId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Heartbeat ingestion owns all learning work. Launch requests only read cached allowances. */
|
||||
export function recordInstanceResources(hostId: string, inventory: HostInventory, timestamp: string): void {
|
||||
if (!inventory.instanceResources?.length || !inventory.dockerRunning || !inventory.cpuCount || !inventory.memoryTotal) return;
|
||||
export function recordInstanceResources(
|
||||
hostId: string,
|
||||
inventory: HostInventory,
|
||||
timestamp: string
|
||||
): void {
|
||||
if (
|
||||
!inventory.instanceResources?.length ||
|
||||
!inventory.dockerRunning ||
|
||||
!inventory.cpuCount ||
|
||||
!inventory.memoryTotal
|
||||
)
|
||||
return;
|
||||
const handle = database();
|
||||
const bucket = timestamp.slice(0, 16) + ':00.000Z';
|
||||
const observed = new Map<Phase, { cpu: number; memory: number }>();
|
||||
const owned = handle.prepare('SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL');
|
||||
const owned = handle.prepare(
|
||||
'SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL'
|
||||
);
|
||||
const insert = handle.prepare(`INSERT INTO ec_instance_resource_samples
|
||||
(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,?,?,?,?,?)
|
||||
ON CONFLICT(host_id,instance_id,generation,phase,bucket) DO UPDATE SET
|
||||
@@ -71,58 +132,125 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory
|
||||
// retired and delayed pre-rebuild samples cannot contaminate the model.
|
||||
if (!owned.get(sample.instanceId, hostId, sample.generation)) continue;
|
||||
const phase: Phase = sample.uptimeSeconds < 300 ? 'startup' : 'running';
|
||||
insert.run(hostId, sample.instanceId, sample.generation, phase, bucket, sample.cpuCores, sample.memoryBytes);
|
||||
insert.run(
|
||||
hostId,
|
||||
sample.instanceId,
|
||||
sample.generation,
|
||||
phase,
|
||||
bucket,
|
||||
sample.cpuCores,
|
||||
sample.memoryBytes
|
||||
);
|
||||
const prior = observed.get(phase);
|
||||
observed.set(phase, { cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) });
|
||||
observed.set(phase, {
|
||||
cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores),
|
||||
memory: Math.max(prior?.memory ?? 0, sample.memoryBytes)
|
||||
});
|
||||
}
|
||||
for (const [phase, peak] of observed) for (const scope of [hostId, '*']) {
|
||||
const previous = handle.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?').get(scope, phase) as AllowanceRow | undefined;
|
||||
if (!previous || previous.computed_at < bucket) {
|
||||
const rows = handle.prepare(`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples
|
||||
WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
|
||||
.all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
|
||||
{ cpu_cores: number; memory_bytes: number; bucket: string }[];
|
||||
const percentile = (values: number[]) => values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1];
|
||||
const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu);
|
||||
const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory));
|
||||
handle.prepare(`INSERT INTO ec_resource_allowances
|
||||
for (const [phase, peak] of observed)
|
||||
for (const scope of [hostId, '*']) {
|
||||
const previous = handle
|
||||
.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?')
|
||||
.get(scope, phase) as AllowanceRow | undefined;
|
||||
if (!previous || previous.computed_at < bucket) {
|
||||
const rows = handle
|
||||
.prepare(
|
||||
`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples
|
||||
WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`
|
||||
)
|
||||
.all(
|
||||
...(scope === '*' ? [] : [scope]),
|
||||
phase,
|
||||
new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(),
|
||||
MAX_SAMPLES
|
||||
) as { cpu_cores: number; memory_bytes: number; bucket: string }[];
|
||||
const percentile = (values: number[]) =>
|
||||
values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1];
|
||||
const cpu = Math.max(percentile(rows.map((row) => row.cpu_cores)), peak.cpu);
|
||||
const memory = Math.ceil(
|
||||
Math.max(percentile(rows.map((row) => row.memory_bytes)), peak.memory)
|
||||
);
|
||||
handle
|
||||
.prepare(
|
||||
`INSERT INTO ec_resource_allowances
|
||||
(scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET
|
||||
cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples,
|
||||
window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`)
|
||||
.run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp);
|
||||
if (scope === '*' && fleetSettings().instanceResourceKeepDays) handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket<?')
|
||||
.run(new Date(Date.parse(timestamp) - Math.max(WINDOW_MS, fleetSettings().instanceResourceKeepDays * 86400_000)).toISOString());
|
||||
} else {
|
||||
// Raise the allowance immediately on a new peak; a lower sample
|
||||
// cannot reduce it until the next scheduled percentile calculation.
|
||||
handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`)
|
||||
.run(peak.cpu, Math.ceil(peak.memory), timestamp, scope, phase);
|
||||
window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`
|
||||
)
|
||||
.run(
|
||||
scope,
|
||||
phase,
|
||||
cpu,
|
||||
memory,
|
||||
rows.length,
|
||||
rows.at(-1)!.bucket,
|
||||
timestamp,
|
||||
timestamp
|
||||
);
|
||||
if (scope === '*' && fleetSettings().instanceResourceKeepDays)
|
||||
handle
|
||||
.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket<?')
|
||||
.run(
|
||||
new Date(
|
||||
Date.parse(timestamp) -
|
||||
Math.max(
|
||||
WINDOW_MS,
|
||||
fleetSettings().instanceResourceKeepDays * 86400_000
|
||||
)
|
||||
).toISOString()
|
||||
);
|
||||
} else {
|
||||
// Raise the allowance immediately on a new peak; a lower sample
|
||||
// cannot reduce it until the next scheduled percentile calculation.
|
||||
handle
|
||||
.prepare(
|
||||
`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`
|
||||
)
|
||||
.run(peak.cpu, Math.ceil(peak.memory), timestamp, scope, phase);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Host measurements account for hardware differences; fleet evidence covers new hosts. */
|
||||
export function capacityCalibration(hostId: string): CapacityCalibration {
|
||||
const rows = database().prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?')
|
||||
const rows = database()
|
||||
.prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?')
|
||||
.all(hostId, '*', new Date(Date.now() - WINDOW_MS).toISOString()) as AllowanceRow[];
|
||||
const allowance = (phase: Phase): ResourceAllowance => {
|
||||
const local = rows.find(row => row.scope === hostId && row.phase === phase);
|
||||
const fleet = rows.find(row => row.scope === '*' && row.phase === phase);
|
||||
const learned = (row?: AllowanceRow) => !!row && row.samples >= (phase === 'startup' ? 3 : 20)
|
||||
&& Date.parse(row.observed_at) - Date.parse(row.window_start) >= (phase === 'startup' ? 120_000 : 300_000);
|
||||
const local = rows.find((row) => row.scope === hostId && row.phase === phase);
|
||||
const fleet = rows.find((row) => row.scope === '*' && row.phase === phase);
|
||||
const learned = (row?: AllowanceRow) =>
|
||||
!!row &&
|
||||
row.samples >= (phase === 'startup' ? 3 : 20) &&
|
||||
Date.parse(row.observed_at) - Date.parse(row.window_start) >=
|
||||
(phase === 'startup' ? 120_000 : 300_000);
|
||||
const selected = learned(local) ? local : learned(fleet) ? fleet : undefined;
|
||||
return {
|
||||
// A few higher observations can increase a fleet/bootstrap allowance
|
||||
// immediately. Decreasing it requires the evidence thresholds above.
|
||||
cpuCores: Math.max(0.01, selected?.cpu_cores ?? BOOTSTRAP_CPU, local?.cpu_cores ?? 0, selected ? 0 : fleet?.cpu_cores ?? 0),
|
||||
memoryBytes: Math.max(selected?.memory_bytes ?? BOOTSTRAP_MEMORY, local?.memory_bytes ?? 0, selected ? 0 : fleet?.memory_bytes ?? 0),
|
||||
source: selected ? selected.scope === hostId ? 'host' : 'fleet' : 'bootstrap',
|
||||
cpuCores: Math.max(
|
||||
0.01,
|
||||
selected?.cpu_cores ?? BOOTSTRAP_CPU,
|
||||
local?.cpu_cores ?? 0,
|
||||
selected ? 0 : (fleet?.cpu_cores ?? 0)
|
||||
),
|
||||
memoryBytes: Math.max(
|
||||
selected?.memory_bytes ?? BOOTSTRAP_MEMORY,
|
||||
local?.memory_bytes ?? 0,
|
||||
selected ? 0 : (fleet?.memory_bytes ?? 0)
|
||||
),
|
||||
source: selected ? (selected.scope === hostId ? 'host' : 'fleet') : 'bootstrap',
|
||||
samples: selected?.samples ?? local?.samples ?? fleet?.samples ?? 0,
|
||||
observedAt: selected?.observed_at ?? local?.observed_at ?? fleet?.observed_at ?? null
|
||||
};
|
||||
};
|
||||
const running = allowance('running'); const startup = allowance('startup');
|
||||
if (running.source === 'bootstrap' && (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)) startup.source = 'bootstrap';
|
||||
const running = allowance('running');
|
||||
const startup = allowance('startup');
|
||||
if (
|
||||
running.source === 'bootstrap' &&
|
||||
(running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)
|
||||
)
|
||||
startup.source = 'bootstrap';
|
||||
startup.cpuCores = Math.max(startup.cpuCores, running.cpuCores);
|
||||
startup.memoryBytes = Math.max(startup.memoryBytes, running.memoryBytes);
|
||||
return { running, startup };
|
||||
|
||||
@@ -15,8 +15,11 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
|
||||
const { duration, bucketMs } = windows[query.range];
|
||||
const startAt = now - duration;
|
||||
const count = duration / bucketMs;
|
||||
const hosts = handle.prepare(`SELECT id,created_at FROM ec_hosts
|
||||
WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`)
|
||||
const hosts = handle
|
||||
.prepare(
|
||||
`SELECT id,created_at FROM ec_hosts
|
||||
WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`
|
||||
)
|
||||
.all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[];
|
||||
// The primary key bounds each host scan to the selected interval. Aggregate
|
||||
// before crossing the worker boundary; do not return raw heartbeat rows.
|
||||
@@ -42,8 +45,14 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
|
||||
for (const point of points) {
|
||||
if (createdAt <= Math.min(now, point.at + bucketMs)) point.expectedHosts++;
|
||||
}
|
||||
const buckets = samples.all(count - 1, startAt, bucketMs, host.id,
|
||||
new Date(startAt).toISOString(), new Date(now).toISOString()) as { bucket: number; cpu: number; memory: number }[];
|
||||
const buckets = samples.all(
|
||||
count - 1,
|
||||
startAt,
|
||||
bucketMs,
|
||||
host.id,
|
||||
new Date(startAt).toISOString(),
|
||||
new Date(now).toISOString()
|
||||
) as { bucket: number; cpu: number; memory: number }[];
|
||||
for (const bucket of buckets) {
|
||||
const point = points[bucket.bucket];
|
||||
if (!point || createdAt > Math.min(now, point.at + bucketMs)) continue;
|
||||
|
||||
@@ -14,56 +14,161 @@ export async function readCapacityCurrent(query: CapacityQuery): Promise<Capacit
|
||||
await yieldToController();
|
||||
snapshot = readAvailabilitySnapshot();
|
||||
}
|
||||
if (!snapshot) return { updatedAt: null, hosts: [], hostOptions: [], warnings: ['Capacity measurements are updating.'],
|
||||
summary: { estimatedServers: null, cpuCoresAvailable: null, memoryBytesAvailable: null, portsAvailable: null,
|
||||
eligibleHosts: 0, totalHosts: 0, startupSlots: null, startupLimit: null, activeStartups: null, queued: null } };
|
||||
if (!snapshot)
|
||||
return {
|
||||
updatedAt: null,
|
||||
hosts: [],
|
||||
hostOptions: [],
|
||||
warnings: ['Capacity measurements are updating.'],
|
||||
summary: {
|
||||
estimatedServers: null,
|
||||
cpuCoresAvailable: null,
|
||||
memoryBytesAvailable: null,
|
||||
portsAvailable: null,
|
||||
eligibleHosts: 0,
|
||||
totalHosts: 0,
|
||||
startupSlots: null,
|
||||
startupLimit: null,
|
||||
activeStartups: null,
|
||||
queued: null
|
||||
}
|
||||
};
|
||||
const libraries = { t5: newestReadyLibraryId('t5'), t6: newestReadyLibraryId('t6') };
|
||||
const requestedGames = query.game === 'all' ? ['t5', 't6'] as const : [query.game];
|
||||
const requestedGames = query.game === 'all' ? (['t5', 't6'] as const) : [query.game];
|
||||
const now = Date.now();
|
||||
const hosts: CapacityHost[] = [...snapshot.hosts.values()].filter(({ host }) => !query.hostId || host.id === query.hostId).map(observation => {
|
||||
const { host, calibration } = observation;
|
||||
const fresh = !!host.lastHeartbeat && now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS;
|
||||
const connected = host.connected && fresh;
|
||||
const games = (['t5', 't6'] as const).filter(game => libraries[game] && host.inventory.libraries?.some(
|
||||
library => library.id === libraries[game] && ['ready', 'outdated'].includes(library.state)));
|
||||
const reasons: string[] = [];
|
||||
if (!connected) reasons.push(host.enrolled ? 'Waiting for a fresh host report.' : 'This host has not enrolled yet.');
|
||||
if (host.mode !== 'active') reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.');
|
||||
if (host.decommission) reasons.push('Host is being removed.');
|
||||
if (!host.credentialActive && host.enrolled) reasons.push('Host credentials are inactive.');
|
||||
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
|
||||
if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage)) reasons.push('The default runtime is not ready.');
|
||||
if (!requestedGames.some(game => games.includes(game))) reasons.push('Latest game files for this selection are not ready.');
|
||||
if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.');
|
||||
const eligible = connected && host.readiness === 'ready' && reasons.length === 0;
|
||||
const cpuCoresAvailable = fresh ? observation.freeCpu : null;
|
||||
const memoryBytesAvailable = fresh ? observation.freeMemory : null;
|
||||
const portsAvailable = freePorts(observation);
|
||||
const limits = { cpu: cpuCoresAvailable === null ? null : Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)),
|
||||
memory: memoryBytesAvailable === null ? null : Math.max(0, Math.floor(memoryBytesAvailable / calibration.running.memoryBytes)),
|
||||
ports: portsAvailable };
|
||||
const estimatedServers = !eligible ? 0 : limits.cpu === null || limits.memory === null ? null : Math.min(limits.cpu, limits.memory, limits.ports);
|
||||
const limitingResource: CapacityHost['limitingResource'] = estimatedServers === null || !eligible ? null
|
||||
: limits.ports === estimatedServers ? 'Ports' : limits.memory === estimatedServers ? 'Memory' : 'CPU';
|
||||
if (eligible && (limits.cpu === null || limits.memory === null)) reasons.push('Waiting for valid CPU and memory measurements.');
|
||||
else if (eligible && estimatedServers === 0) reasons.push(`No additional instances currently fit: ${limitingResource?.toLowerCase()} capacity is full.`);
|
||||
return { id: host.id, name: host.name, readiness: host.readiness, connected, updatedAt: host.lastHeartbeat,
|
||||
eligible, games, reasons: [...new Set(reasons)], cpuCoresAvailable, memoryBytesAvailable,
|
||||
portsAvailable, totalPorts: host.portMax - host.portMin + 1, estimatedServers, limits, limitingResource, calibration };
|
||||
}).sort((first, second) => first.name.localeCompare(second.name));
|
||||
const eligible = hosts.filter(host => host.eligible);
|
||||
const hosts: CapacityHost[] = [...snapshot.hosts.values()]
|
||||
.filter(({ host }) => !query.hostId || host.id === query.hostId)
|
||||
.map((observation) => {
|
||||
const { host, calibration } = observation;
|
||||
const fresh =
|
||||
!!host.lastHeartbeat &&
|
||||
now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS;
|
||||
const connected = host.connected && fresh;
|
||||
const games = (['t5', 't6'] as const).filter(
|
||||
(game) =>
|
||||
libraries[game] &&
|
||||
host.inventory.libraries?.some(
|
||||
(library) =>
|
||||
library.id === libraries[game] &&
|
||||
['ready', 'outdated'].includes(library.state)
|
||||
)
|
||||
);
|
||||
const reasons: string[] = [];
|
||||
if (!connected)
|
||||
reasons.push(
|
||||
host.enrolled
|
||||
? 'Waiting for a fresh host report.'
|
||||
: 'This host has not enrolled yet.'
|
||||
);
|
||||
if (host.mode !== 'active')
|
||||
reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.');
|
||||
if (host.decommission) reasons.push('Host is being removed.');
|
||||
if (!host.credentialActive && host.enrolled)
|
||||
reasons.push('Host credentials are inactive.');
|
||||
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
|
||||
if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage))
|
||||
reasons.push('The default runtime is not ready.');
|
||||
if (!requestedGames.some((game) => games.includes(game)))
|
||||
reasons.push('Latest game files for this selection are not ready.');
|
||||
if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3)
|
||||
reasons.push('Storage is below the launch reserve.');
|
||||
const eligible = connected && host.readiness === 'ready' && reasons.length === 0;
|
||||
const cpuCoresAvailable = fresh ? observation.freeCpu : null;
|
||||
const memoryBytesAvailable = fresh ? observation.freeMemory : null;
|
||||
const portsAvailable = freePorts(observation);
|
||||
const limits = {
|
||||
cpu:
|
||||
cpuCoresAvailable === null
|
||||
? null
|
||||
: Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)),
|
||||
memory:
|
||||
memoryBytesAvailable === null
|
||||
? null
|
||||
: Math.max(
|
||||
0,
|
||||
Math.floor(memoryBytesAvailable / calibration.running.memoryBytes)
|
||||
),
|
||||
ports: portsAvailable
|
||||
};
|
||||
const estimatedServers = !eligible
|
||||
? 0
|
||||
: limits.cpu === null || limits.memory === null
|
||||
? null
|
||||
: Math.min(limits.cpu, limits.memory, limits.ports);
|
||||
const limitingResource: CapacityHost['limitingResource'] =
|
||||
estimatedServers === null || !eligible
|
||||
? null
|
||||
: limits.ports === estimatedServers
|
||||
? 'Ports'
|
||||
: limits.memory === estimatedServers
|
||||
? 'Memory'
|
||||
: 'CPU';
|
||||
if (eligible && (limits.cpu === null || limits.memory === null))
|
||||
reasons.push('Waiting for valid CPU and memory measurements.');
|
||||
else if (eligible && estimatedServers === 0)
|
||||
reasons.push(
|
||||
`No additional instances currently fit: ${limitingResource?.toLowerCase()} capacity is full.`
|
||||
);
|
||||
return {
|
||||
id: host.id,
|
||||
name: host.name,
|
||||
readiness: host.readiness,
|
||||
connected,
|
||||
updatedAt: host.lastHeartbeat,
|
||||
eligible,
|
||||
games,
|
||||
reasons: [...new Set(reasons)],
|
||||
cpuCoresAvailable,
|
||||
memoryBytesAvailable,
|
||||
portsAvailable,
|
||||
totalPorts: host.portMax - host.portMin + 1,
|
||||
estimatedServers,
|
||||
limits,
|
||||
limitingResource,
|
||||
calibration
|
||||
};
|
||||
})
|
||||
.sort((first, second) => first.name.localeCompare(second.name));
|
||||
const eligible = hosts.filter((host) => host.eligible);
|
||||
const sum = (key: 'cpuCoresAvailable' | 'memoryBytesAvailable' | 'estimatedServers') =>
|
||||
eligible.some(host => host[key] === null) ? null : eligible.reduce((total, host) => total + (host[key] ?? 0), 0);
|
||||
eligible.some((host) => host[key] === null)
|
||||
? null
|
||||
: eligible.reduce((total, host) => total + (host[key] ?? 0), 0);
|
||||
const warnings: string[] = [];
|
||||
if (!hosts.length) warnings.push(query.hostId ? 'The selected host is unavailable.' : 'Add a host to start tracking launch capacity.');
|
||||
else if (!eligible.length) warnings.push('No host is ready for this game selection. Review the host details below.');
|
||||
else if (eligible.length < hosts.length) warnings.push(`${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.`);
|
||||
if (eligible.some(host => host.estimatedServers === null)) warnings.push('Some resource measurements are missing; the total estimate is unknown.');
|
||||
if (!requestedGames.some(game => libraries[game])) warnings.push('Upload game files before launching this selection.');
|
||||
return { updatedAt: snapshot.updatedAt, hosts,
|
||||
hostOptions: [...snapshot.hosts.values()].map(({ host }) => ({ id: host.id, name: host.name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
warnings, summary: { estimatedServers: sum('estimatedServers'), cpuCoresAvailable: sum('cpuCoresAvailable'),
|
||||
memoryBytesAvailable: sum('memoryBytesAvailable'), portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0),
|
||||
eligibleHosts: eligible.length, totalHosts: hosts.length, startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active),
|
||||
startupLimit: snapshot.startupLimit, activeStartups: snapshot.active, queued: snapshot.queued } };
|
||||
if (!hosts.length)
|
||||
warnings.push(
|
||||
query.hostId
|
||||
? 'The selected host is unavailable.'
|
||||
: 'Add a host to start tracking launch capacity.'
|
||||
);
|
||||
else if (!eligible.length)
|
||||
warnings.push('No host is ready for this game selection. Review the host details below.');
|
||||
else if (eligible.length < hosts.length)
|
||||
warnings.push(
|
||||
`${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.`
|
||||
);
|
||||
if (eligible.some((host) => host.estimatedServers === null))
|
||||
warnings.push('Some resource measurements are missing; the total estimate is unknown.');
|
||||
if (!requestedGames.some((game) => libraries[game]))
|
||||
warnings.push('Upload game files before launching this selection.');
|
||||
return {
|
||||
updatedAt: snapshot.updatedAt,
|
||||
hosts,
|
||||
hostOptions: [...snapshot.hosts.values()]
|
||||
.map(({ host }) => ({ id: host.id, name: host.name }))
|
||||
.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
warnings,
|
||||
summary: {
|
||||
estimatedServers: sum('estimatedServers'),
|
||||
cpuCoresAvailable: sum('cpuCoresAvailable'),
|
||||
memoryBytesAvailable: sum('memoryBytesAvailable'),
|
||||
portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0),
|
||||
eligibleHosts: eligible.length,
|
||||
totalHosts: hosts.length,
|
||||
startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active),
|
||||
startupLimit: snapshot.startupLimit,
|
||||
activeStartups: snapshot.active,
|
||||
queued: snapshot.queued
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { CapacityData, CapacityQuery } from '../../capacity.js';
|
||||
|
||||
export async function capacityPage(query: CapacityQuery): Promise<CapacityData> {
|
||||
const current = await readCapacityCurrent(query);
|
||||
if (query.hostId && !current.hostOptions.some(host => host.id === query.hostId)) error(404, 'Host not found.');
|
||||
if (query.hostId && !current.hostOptions.some((host) => host.id === query.hostId))
|
||||
error(404, 'Host not found.');
|
||||
return { query, current, history: await capacityHistoryReport(query) };
|
||||
}
|
||||
|
||||
@@ -10,9 +10,13 @@ const MEMORY_HEADROOM = 500_000_000;
|
||||
const CPU_USABLE_FRACTION = 0.95;
|
||||
export class CapacityError extends Error {}
|
||||
export interface PlacementOptions {
|
||||
hostId?: string; libraryId: string; image: string; bindAddress?: string;
|
||||
hostId?: string;
|
||||
libraryId: string;
|
||||
image: string;
|
||||
bindAddress?: string;
|
||||
excludeHostIds?: string[];
|
||||
thorough?: boolean; excludeInstanceId?: string;
|
||||
thorough?: boolean;
|
||||
excludeInstanceId?: string;
|
||||
// Execution only: not-yet-submitted launches wait their turn instead of
|
||||
// requiring their entire backlog to fit before any one can start.
|
||||
dispatch?: boolean;
|
||||
@@ -20,12 +24,24 @@ export interface PlacementOptions {
|
||||
}
|
||||
|
||||
/** One indexed port read per host, including both protocols and wildcard binds. */
|
||||
export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}): number | null {
|
||||
export function availableHostPort(
|
||||
host: Host,
|
||||
options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}
|
||||
): number | null {
|
||||
const bind = options.bindAddress ?? host.bindAddress;
|
||||
const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
|
||||
const rows = db()
|
||||
.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
|
||||
.all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[];
|
||||
const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind)
|
||||
|| ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port));
|
||||
const used = new Set(
|
||||
rows
|
||||
.filter(
|
||||
(row) =>
|
||||
row.bind_address === bind ||
|
||||
['0.0.0.0', '::'].includes(bind) ||
|
||||
['0.0.0.0', '::'].includes(row.bind_address)
|
||||
)
|
||||
.map((row) => row.port)
|
||||
);
|
||||
for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port);
|
||||
for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port;
|
||||
return null;
|
||||
@@ -33,38 +49,66 @@ export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'b
|
||||
|
||||
function projectedLoad(host: Host, options: PlacementOptions): number {
|
||||
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
|
||||
if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) {
|
||||
if (
|
||||
!cpuCount ||
|
||||
cpuUsagePercent === undefined ||
|
||||
!memoryTotal ||
|
||||
memoryAvailable === undefined ||
|
||||
memoryAvailable > memoryTotal
|
||||
) {
|
||||
throw new Error('Waiting for valid CPU and memory measurements.');
|
||||
}
|
||||
let historicalCpu = cpuUsagePercent;
|
||||
let historicalAvailable = memoryAvailable;
|
||||
if (options.thorough) {
|
||||
const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
|
||||
FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`)
|
||||
.get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null };
|
||||
const history = db()
|
||||
.prepare(
|
||||
`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
|
||||
FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`
|
||||
)
|
||||
.get(
|
||||
host.id,
|
||||
new Date(Date.now() - 5 * 60_000).toISOString(),
|
||||
cpuCount,
|
||||
memoryTotal
|
||||
) as { cpu: number | null; memory: number | null };
|
||||
historicalCpu = Math.max(cpuUsagePercent, history.cpu ?? cpuUsagePercent);
|
||||
historicalAvailable = Math.min(memoryAvailable, history.memory ?? memoryAvailable);
|
||||
}
|
||||
const instances = db().prepare(`SELECT i.id, i.generation,
|
||||
const instances = db()
|
||||
.prepare(
|
||||
`SELECT i.id, i.generation,
|
||||
NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
||||
WHERE n.instance_id=i.id AND n.generation=i.generation AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
|
||||
EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
||||
WHERE n.instance_id=i.id AND d.action='create' AND d.state='queued' AND d.submitted_at IS NULL
|
||||
AND n.operation_id IS NULL AND n.stage_operation_id IS NULL) AS unsubmitted
|
||||
FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
|
||||
.all(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as
|
||||
{ id: string; generation: number; pending: number; unsubmitted: number }[];
|
||||
FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`
|
||||
)
|
||||
.all(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as {
|
||||
id: string;
|
||||
generation: number;
|
||||
pending: number;
|
||||
unsubmitted: number;
|
||||
}[];
|
||||
const { running, startup } = capacityCalibration(host.id);
|
||||
const incoming = db().prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations
|
||||
WHERE target_host_id=? AND removed_at IS NULL AND instance_id!=?`).get(host.id, options.excludeInstanceId ?? '') as { n: number };
|
||||
const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
|
||||
const incoming = db()
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS n FROM ec_host_migrations
|
||||
WHERE target_host_id=? AND removed_at IS NULL AND instance_id!=?`
|
||||
)
|
||||
.get(host.id, options.excludeInstanceId ?? '') as { n: number };
|
||||
const planned = options.planned?.filter((item) => item.hostId === host.id).length ?? 0;
|
||||
// Admission budgets the eventual running collection. Dispatch budgets only
|
||||
// the next startup; the rest of the queue waits for live capacity to recover.
|
||||
const launch = options.dispatch ? startup : running;
|
||||
let extraCpu = planned * running.cpuCores + launch.cpuCores + incoming.n * startup.cpuCores;
|
||||
let extraMemory = planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes;
|
||||
let extraMemory =
|
||||
planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes;
|
||||
let reservations = planned + 1 + incoming.n;
|
||||
const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
|
||||
const measurements = new Map(
|
||||
host.inventory.instanceResources?.map((sample) => [sample.instanceId, sample])
|
||||
);
|
||||
for (const instance of instances) {
|
||||
if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue;
|
||||
if (instance.unsubmitted) {
|
||||
@@ -82,28 +126,45 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
|
||||
extraMemory += startup.memoryBytes;
|
||||
reservations++;
|
||||
}
|
||||
const cores = cpuCount * cpuUsagePercent / 100 + extraCpu;
|
||||
const cores = (cpuCount * cpuUsagePercent) / 100 + extraCpu;
|
||||
const usedMemory = memoryTotal - memoryAvailable;
|
||||
const memory = usedMemory + extraMemory;
|
||||
const usableMemory = memoryTotal - MEMORY_HEADROOM;
|
||||
const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
|
||||
if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`);
|
||||
if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. `
|
||||
+ `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).`);
|
||||
if (cores > cpuCount * CPU_USABLE_FRACTION)
|
||||
throw new Error(
|
||||
`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`
|
||||
);
|
||||
if (memory > usableMemory)
|
||||
throw new Error(
|
||||
`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. ` +
|
||||
`${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).`
|
||||
);
|
||||
// Recovered resources must not remain unavailable until the historical
|
||||
// average catches up. History ranks hosts after current capacity passes.
|
||||
const historicalMemory = memoryTotal - historicalAvailable + extraMemory;
|
||||
const historicalCores = cpuCount * historicalCpu / 100 + extraCpu;
|
||||
const historicalCores = (cpuCount * historicalCpu) / 100 + extraCpu;
|
||||
return Math.max(historicalCores / cpuCount, historicalMemory / memoryTotal);
|
||||
}
|
||||
|
||||
/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */
|
||||
export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } {
|
||||
export function choosePlacement(options: PlacementOptions): {
|
||||
host: Host;
|
||||
port: number;
|
||||
load: number;
|
||||
} {
|
||||
const candidates: { host: Host; port: number; load: number }[] = [];
|
||||
const reasons: string[] = [];
|
||||
for (const host of listHosts().filter(host => (!options.hostId || host.id === options.hostId) && !options.excludeHostIds?.includes(host.id))) {
|
||||
for (const host of listHosts().filter(
|
||||
(host) =>
|
||||
(!options.hostId || host.id === options.hostId) &&
|
||||
!options.excludeHostIds?.includes(host.id)
|
||||
)) {
|
||||
try {
|
||||
requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
|
||||
requireReadyHost(host.id, {
|
||||
image: options.image,
|
||||
requiredBytes: INSTANCE_STORAGE_BYTES
|
||||
});
|
||||
assertReadyLibrary(host.id, options.libraryId);
|
||||
const port = availableHostPort(host, options);
|
||||
if (port === null) throw new Error('No available UDP/TCP game port.');
|
||||
@@ -111,9 +172,13 @@ export function choosePlacement(options: PlacementOptions): { host: Host; port:
|
||||
const candidate = { host, port, load };
|
||||
if (!options.thorough) return candidate;
|
||||
candidates.push(candidate);
|
||||
} catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); }
|
||||
} catch (error) {
|
||||
reasons.push(`${host.name}: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id));
|
||||
if (candidates.length) return candidates[0];
|
||||
throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`);
|
||||
throw new CapacityError(
|
||||
`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`
|
||||
);
|
||||
}
|
||||
|
||||
@@ -6,14 +6,23 @@ const CHUNK_CHARACTERS = 16 * 1024;
|
||||
const CHUNK_BYTES = 64 * 1024;
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
function* jsonParts(operations: OperationDeliveryRow[], serverTime: string): Generator<string | Buffer> {
|
||||
function* jsonParts(
|
||||
operations: OperationDeliveryRow[],
|
||||
serverTime: string
|
||||
): Generator<string | Buffer> {
|
||||
yield '{"operations":[';
|
||||
for (const [index, operation] of operations.entries()) {
|
||||
if (index) yield ',';
|
||||
const metadata = JSON.stringify({
|
||||
id: operation.id, hostId: operation.host_id, kind: operation.kind,
|
||||
instanceId: operation.instance_id, generation: operation.generation, state: operation.state,
|
||||
error: operation.error, createdAt: operation.created_at, updatedAt: operation.updated_at
|
||||
id: operation.id,
|
||||
hostId: operation.host_id,
|
||||
kind: operation.kind,
|
||||
instanceId: operation.instance_id,
|
||||
generation: operation.generation,
|
||||
state: operation.state,
|
||||
error: operation.error,
|
||||
createdAt: operation.created_at,
|
||||
updatedAt: operation.updated_at
|
||||
});
|
||||
yield metadata.slice(0, -1) + ',"payload":';
|
||||
// Load only the immutable payload currently being sent, after the
|
||||
@@ -26,10 +35,14 @@ function* jsonParts(operations: OperationDeliveryRow[], serverTime: string): Gen
|
||||
yield '],"serverTime":' + JSON.stringify(serverTime) + '}';
|
||||
}
|
||||
|
||||
function* encodedChunks(operations: OperationDeliveryRow[], serverTime: string): Generator<Uint8Array> {
|
||||
function* encodedChunks(
|
||||
operations: OperationDeliveryRow[],
|
||||
serverTime: string
|
||||
): Generator<Uint8Array> {
|
||||
for (const part of jsonParts(operations, serverTime)) {
|
||||
if (Buffer.isBuffer(part)) {
|
||||
for (let start = 0; start < part.length; start += CHUNK_BYTES) yield part.subarray(start, start + CHUNK_BYTES);
|
||||
for (let start = 0; start < part.length; start += CHUNK_BYTES)
|
||||
yield part.subarray(start, start + CHUNK_BYTES);
|
||||
continue;
|
||||
}
|
||||
for (let start = 0; start < part.length;) {
|
||||
@@ -45,7 +58,10 @@ function* encodedChunks(operations: OperationDeliveryRow[], serverTime: string):
|
||||
}
|
||||
|
||||
/** Bound encoding work and yield between chunks so health requests keep flowing. */
|
||||
export function hostOperationsResponse(operations: OperationDeliveryRow[], serverTime = new Date().toISOString()): Response {
|
||||
export function hostOperationsResponse(
|
||||
operations: OperationDeliveryRow[],
|
||||
serverTime = new Date().toISOString()
|
||||
): Response {
|
||||
const chunks = encodedChunks(operations, serverTime);
|
||||
let cancelled = false;
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
@@ -56,7 +72,12 @@ export function hostOperationsResponse(operations: OperationDeliveryRow[], serve
|
||||
if (next.done) controller.close();
|
||||
else controller.enqueue(next.value);
|
||||
},
|
||||
cancel() { cancelled = true; chunks.return(undefined); }
|
||||
cancel() {
|
||||
cancelled = true;
|
||||
chunks.return(undefined);
|
||||
}
|
||||
});
|
||||
return new Response(stream, {
|
||||
headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' }
|
||||
});
|
||||
return new Response(stream, { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' } });
|
||||
}
|
||||
|
||||
@@ -6,7 +6,12 @@ const visible = (alias: string) => `${alias}.history_expired_at IS NULL`;
|
||||
|
||||
/** Durable totals for global, host, server, and host/server filters; receipts update only changed membership. */
|
||||
export function installHistoryCounts(handle: Database.Database): void {
|
||||
if (handle.prepare("SELECT 1 FROM ec_host_schema_migrations WHERE name='history-counts-v1'").get()) return;
|
||||
if (
|
||||
handle
|
||||
.prepare("SELECT 1 FROM ec_host_schema_migrations WHERE name='history-counts-v1'")
|
||||
.get()
|
||||
)
|
||||
return;
|
||||
handle.transaction(() => {
|
||||
handle.exec(`CREATE TABLE ec_host_operation_history_counts (
|
||||
host_id TEXT NOT NULL, server_name TEXT NOT NULL, total INTEGER NOT NULL,
|
||||
@@ -15,7 +20,12 @@ export function installHistoryCounts(handle: Database.Database): void {
|
||||
DROP INDEX ec_host_operations_important_history;
|
||||
CREATE INDEX ec_host_operations_important_history ON ec_host_operations(created_at)
|
||||
WHERE history_expired_at IS NULL AND ${importantOperationSql.replaceAll('o.', '')};`);
|
||||
for (const [host, server] of [['', ''], ['o.host_id', ''], ['', 'o.server_name'], ['o.host_id', 'o.server_name']]) {
|
||||
for (const [host, server] of [
|
||||
['', ''],
|
||||
['o.host_id', ''],
|
||||
['', 'o.server_name'],
|
||||
['o.host_id', 'o.server_name']
|
||||
]) {
|
||||
const groups = [host, server].filter(Boolean);
|
||||
handle.exec(`INSERT INTO ec_host_operation_history_counts
|
||||
SELECT ${host || "''"},${server || "''"},COUNT(*),COALESCE(SUM(o.routine=1),0),COALESCE(SUM(${important('o')}),0)
|
||||
@@ -37,6 +47,8 @@ export function installHistoryCounts(handle: Database.Database): void {
|
||||
WHEN OLD.host_id IS NOT NEW.host_id OR OLD.server_name IS NOT NEW.server_name OR OLD.routine IS NOT NEW.routine
|
||||
OR (${visible('OLD')}) IS NOT (${visible('NEW')}) OR ${important('OLD')} IS NOT ${important('NEW')}
|
||||
BEGIN ${adjust('OLD', -1)} ${adjust('NEW', 1)} END;`);
|
||||
handle.prepare("INSERT INTO ec_host_schema_migrations(name) VALUES('history-counts-v1')").run();
|
||||
handle
|
||||
.prepare("INSERT INTO ec_host_schema_migrations(name) VALUES('history-counts-v1')")
|
||||
.run();
|
||||
})();
|
||||
}
|
||||
|
||||
@@ -3,7 +3,8 @@ import type { Host } from '../../hosts.js';
|
||||
import { authenticateCredential } from './index.js';
|
||||
|
||||
export function requireAgentTransport(request: Request): void {
|
||||
if (new URL(request.url).protocol !== 'https:' && process.env.EC_AGENT_ALLOW_HTTP !== 'true') error(426, 'Host agents require HTTPS.');
|
||||
if (new URL(request.url).protocol !== 'https:' && process.env.EC_AGENT_ALLOW_HTTP !== 'true')
|
||||
error(426, 'Host agents require HTTPS.');
|
||||
}
|
||||
export function authenticateAgent(request: Request): Host {
|
||||
requireAgentTransport(request);
|
||||
@@ -15,33 +16,56 @@ export function authenticateAgent(request: Request): Host {
|
||||
}
|
||||
export function requireManagementOrigin(request: Request): void {
|
||||
const origin = request.headers.get('origin');
|
||||
if ((origin && origin !== new URL(request.url).origin) || request.headers.get('sec-fetch-site') === 'cross-site') error(403, 'Cross-origin host management requests are not allowed.');
|
||||
if (
|
||||
(origin && origin !== new URL(request.url).origin) ||
|
||||
request.headers.get('sec-fetch-site') === 'cross-site'
|
||||
)
|
||||
error(403, 'Cross-origin host management requests are not allowed.');
|
||||
// Agent credentials only authorize agent endpoints, never host management.
|
||||
if (request.headers.get('authorization')?.match(/^Bearer ec_(?:host|enroll)_/i)) error(403, 'Agent credentials cannot manage the controller.');
|
||||
if (request.headers.get('authorization')?.match(/^Bearer ec_(?:host|enroll)_/i))
|
||||
error(403, 'Agent credentials cannot manage the controller.');
|
||||
}
|
||||
export async function objectBody(request: Request, maxBytes = 1024 * 1024): Promise<Record<string, unknown>> {
|
||||
if (Number(request.headers.get('content-length')) > maxBytes) error(413, 'Request is too large.');
|
||||
export async function objectBody(
|
||||
request: Request,
|
||||
maxBytes = 1024 * 1024
|
||||
): Promise<Record<string, unknown>> {
|
||||
if (Number(request.headers.get('content-length')) > maxBytes)
|
||||
error(413, 'Request is too large.');
|
||||
const reader = request.body?.getReader();
|
||||
if (!reader) error(400, 'Request body must be valid JSON.');
|
||||
const chunks: Uint8Array[] = []; let received = 0;
|
||||
const chunks: Uint8Array[] = [];
|
||||
let received = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
received += value.byteLength;
|
||||
if (received > maxBytes) { void reader.cancel().catch(() => {}); error(413, 'Request is too large.'); }
|
||||
if (received > maxBytes) {
|
||||
void reader.cancel().catch(() => {});
|
||||
error(413, 'Request is too large.');
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} finally { reader.releaseLock(); }
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
const raw = Buffer.concat(chunks, received).toString('utf8');
|
||||
let body: unknown;
|
||||
try { body = JSON.parse(raw); } catch { error(400, 'Request body must be valid JSON.'); }
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) error(400, 'Request body must be an object.');
|
||||
try {
|
||||
body = JSON.parse(raw);
|
||||
} catch {
|
||||
error(400, 'Request body must be valid JSON.');
|
||||
}
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body))
|
||||
error(400, 'Request body must be an object.');
|
||||
return body as Record<string, unknown>;
|
||||
}
|
||||
export function hostApiError(cause: unknown): Response {
|
||||
if (cause && typeof cause === 'object' && 'status' in cause && 'body' in cause) throw cause;
|
||||
return json({ error: cause instanceof Error ? cause.message : 'Host request failed.' }, { status: 400 });
|
||||
return json(
|
||||
{ error: cause instanceof Error ? cause.message : 'Host request failed.' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
export function installerCommand(controller: string, token: string): string {
|
||||
const origin = new URL(controller).origin;
|
||||
|
||||
+1272
-329
File diff suppressed because it is too large
Load Diff
@@ -5,10 +5,14 @@ import { observationReader } from '../fleet/status.js';
|
||||
|
||||
/** Current instances remain visible when their host or game stops reporting. */
|
||||
export function hostInstances(hostId: string): ServerObservation[] {
|
||||
const servers = db().prepare(`SELECT s.* FROM servers s
|
||||
const servers = db()
|
||||
.prepare(
|
||||
`SELECT s.* FROM servers s
|
||||
JOIN native_instances i ON i.server = s.name
|
||||
WHERE i.host_id = ? AND i.retired_at IS NULL AND s.destroyed_at IS NULL
|
||||
ORDER BY s.name`).all(hostId) as ServerRow[];
|
||||
ORDER BY s.name`
|
||||
)
|
||||
.all(hostId) as ServerRow[];
|
||||
if (!servers.length) return [];
|
||||
const observe = observationReader();
|
||||
return servers.map(observe);
|
||||
|
||||
@@ -9,6 +9,8 @@ export function logResultMetadata(value: unknown): Record<string, unknown> | nul
|
||||
...(typeof result.truncated === 'boolean' ? { truncated: result.truncated } : {}),
|
||||
...(typeof result.reset === 'boolean' ? { reset: result.reset } : {}),
|
||||
...(Array.isArray(result.lines) ? { lineCount: result.lines.length } : {}),
|
||||
...(typeof result.contentBase64 === 'string' ? { bytesRead: Buffer.byteLength(result.contentBase64, 'base64') } : {})
|
||||
...(typeof result.contentBase64 === 'string'
|
||||
? { bytesRead: Buffer.byteLength(result.contentBase64, 'base64') }
|
||||
: {})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,23 +1,50 @@
|
||||
export const observationKinds = ['host.inspect', 'instance.inspect', 'rcon.status-batch', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read'];
|
||||
export const observationKinds = [
|
||||
'host.inspect',
|
||||
'instance.inspect',
|
||||
'rcon.status-batch',
|
||||
'files.read',
|
||||
'files.list',
|
||||
'files.stat',
|
||||
'files.manifest',
|
||||
'logs.read'
|
||||
];
|
||||
const commands = ['status', 'getstatus', 'version'];
|
||||
|
||||
export function safeOperationTarget(kind: string, value: string | null): string | null {
|
||||
if (!value || value.length > 1024) return null;
|
||||
const paths = kind === 'files.manifest' ? value.split(', ') : [value];
|
||||
return paths.every(path => path === '.' || !/[\\\x00-\x1f\x7f]/.test(path)
|
||||
&& path.split('/').every(part => part !== '' && part !== '.' && part !== '..')) ? value : null;
|
||||
return paths.every(
|
||||
(path) =>
|
||||
path === '.' ||
|
||||
// eslint-disable-next-line no-control-regex -- Reject control characters in observed paths.
|
||||
(!/[\\\x00-\x1f\x7f]/.test(path) &&
|
||||
path.split('/').every((part) => part !== '' && part !== '.' && part !== '..'))
|
||||
)
|
||||
? value
|
||||
: null;
|
||||
}
|
||||
|
||||
export function successfulObservation(kind: string, payload: Record<string, unknown>, result: unknown): boolean {
|
||||
const routine = observationKinds.includes(kind) || kind === 'rcon.execute' && commands.includes(String(payload.command));
|
||||
const value = result && typeof result === 'object' ? result as Record<string, unknown> : {};
|
||||
export function successfulObservation(
|
||||
kind: string,
|
||||
payload: Record<string, unknown>,
|
||||
result: unknown
|
||||
): boolean {
|
||||
const routine =
|
||||
observationKinds.includes(kind) ||
|
||||
(kind === 'rcon.execute' && commands.includes(String(payload.command)));
|
||||
const value = result && typeof result === 'object' ? (result as Record<string, unknown>) : {};
|
||||
// Older agents set ok=false when any target fails, despite completing the batch.
|
||||
const completedBatch = kind === 'rcon.status-batch' && Array.isArray(value.results);
|
||||
return routine && (completedBatch || value.ok !== false) && value.healthy !== false && (value.error == null || value.error === '');
|
||||
return (
|
||||
routine &&
|
||||
(completedBatch || value.ok !== false) &&
|
||||
value.healthy !== false &&
|
||||
(value.error == null || value.error === '')
|
||||
);
|
||||
}
|
||||
|
||||
/** Uses the operation alias o; serialized payloads may live outside the routing row. */
|
||||
export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')}) THEN 1
|
||||
export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map((kind) => `'${kind}'`).join(',')}) THEN 1
|
||||
WHEN o.kind='rcon.execute' THEN json_extract(COALESCE(
|
||||
(SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')
|
||||
ELSE 0 END)`;
|
||||
@@ -32,7 +59,8 @@ export const successfulObservationSql = `(o.state='completed' AND COALESCE(o.err
|
||||
export const importantOperationSql = `(o.routine=0 OR o.state IN ('failed','unknown')
|
||||
OR (o.state='completed' AND o.observation_success=0) OR COALESCE(o.error,'')!='')`;
|
||||
|
||||
const payloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
|
||||
const payloadSql =
|
||||
'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
|
||||
/** Deliberately exclude arbitrary RCON commands and all file contents/credentials. */
|
||||
export const operationTargetSql = `CASE
|
||||
WHEN o.kind='plutonium.prepare' THEN CASE
|
||||
|
||||
@@ -5,25 +5,44 @@ import { observationReader } from '../fleet/status.js';
|
||||
import type { ServerRow } from '../fleet/servers.js';
|
||||
import type { Host } from '../../hosts.js';
|
||||
|
||||
export interface HostOverview extends Host { serverCount: number; playerCount: number; unreportedInstances: number }
|
||||
export interface HostOverview extends Host {
|
||||
serverCount: number;
|
||||
playerCount: number;
|
||||
unreportedInstances: number;
|
||||
}
|
||||
|
||||
/** Read current placements once and use the fleet's shared telemetry/RCON selection. */
|
||||
export function hostOverview(): HostOverview[] {
|
||||
const hosts = listHosts();
|
||||
const instances = db().prepare(`SELECT s.*, i.host_id FROM native_instances i
|
||||
const instances = db()
|
||||
.prepare(
|
||||
`SELECT s.*, i.host_id FROM native_instances i
|
||||
JOIN servers s ON s.name = i.server
|
||||
WHERE i.retired_at IS NULL AND s.destroyed_at IS NULL`).all() as (ServerRow & { host_id: string })[];
|
||||
const counts = new Map<string, { serverCount: number; playerCount: number; unreportedInstances: number }>();
|
||||
WHERE i.retired_at IS NULL AND s.destroyed_at IS NULL`
|
||||
)
|
||||
.all() as (ServerRow & { host_id: string })[];
|
||||
const counts = new Map<
|
||||
string,
|
||||
{ serverCount: number; playerCount: number; unreportedInstances: number }
|
||||
>();
|
||||
if (instances.length) {
|
||||
const observe = observationReader();
|
||||
for (const instance of instances) {
|
||||
const count = counts.get(instance.host_id) ?? { serverCount: 0, playerCount: 0, unreportedInstances: 0 };
|
||||
const count = counts.get(instance.host_id) ?? {
|
||||
serverCount: 0,
|
||||
playerCount: 0,
|
||||
unreportedInstances: 0
|
||||
};
|
||||
count.serverCount++;
|
||||
const observation = observe(instance);
|
||||
if (observation.health === 'online' && observation.last_players !== null) count.playerCount += observation.last_players;
|
||||
if (observation.health === 'online' && observation.last_players !== null)
|
||||
count.playerCount += observation.last_players;
|
||||
else count.unreportedInstances++;
|
||||
counts.set(instance.host_id, count);
|
||||
}
|
||||
}
|
||||
return hosts.map(host => ({ ...presentHostPreparation(host), ...(counts.get(host.id) ?? { serverCount: 0, playerCount: 0, unreportedInstances: 0 }) }));
|
||||
return hosts.map((host) => ({
|
||||
...presentHostPreparation(host),
|
||||
...(counts.get(host.id) ?? { serverCount: 0, playerCount: 0, unreportedInstances: 0 })
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -1,25 +1,65 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { config } from '../config.js';
|
||||
import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation, shouldDeferPlutoniumPreparation } from './index.js';
|
||||
import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
|
||||
import {
|
||||
AGENT_PROTOCOL_VERSION,
|
||||
getHost,
|
||||
queueHostOperation,
|
||||
shouldDeferPlutoniumPreparation
|
||||
} from './index.js';
|
||||
import {
|
||||
getHostLibraryCache,
|
||||
latestPublishedLibraries,
|
||||
queueLibraryDownload
|
||||
} from '../game-files/index.js';
|
||||
import { GAME_FILE_GAMES } from '../../game-files.js';
|
||||
import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
|
||||
import type {
|
||||
Host,
|
||||
HostOperationState,
|
||||
HostPreparation,
|
||||
HostPreparationItem
|
||||
} from '../../hosts.js';
|
||||
import { scheduleAvailabilityRefresh } from './availability.js';
|
||||
import { readSetting } from '../maintenance/settings.js';
|
||||
import type { PlutoniumRelease } from '../../plutonium.js';
|
||||
|
||||
const activeStates = ['pending', 'accepted', 'running', 'unknown'];
|
||||
const supersededDownload = 'Superseded by newer game files before delivery to the host.';
|
||||
type PreparationOperation = { id: string; sequence: number; last_delivered_at: string | null; state: HostOperationState; updated_at: string; error: string | null };
|
||||
type Target = { item: HostPreparationItem; image?: string; versionId?: string; revision?: number; requiredBytes: number; operation?: PreparationOperation };
|
||||
type PreparationOperation = {
|
||||
id: string;
|
||||
sequence: number;
|
||||
last_delivered_at: string | null;
|
||||
state: HostOperationState;
|
||||
updated_at: string;
|
||||
error: string | null;
|
||||
};
|
||||
type Target = {
|
||||
item: HostPreparationItem;
|
||||
image?: string;
|
||||
versionId?: string;
|
||||
revision?: number;
|
||||
requiredBytes: number;
|
||||
operation?: PreparationOperation;
|
||||
};
|
||||
type QueuedSelection = { libraryId: string };
|
||||
type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
|
||||
type LibraryTarget = Pick<
|
||||
ReturnType<typeof latestPublishedLibraries>[number],
|
||||
'id' | 'sizeBytes' | 'manifestSha256'
|
||||
> & { name: string; queued?: boolean };
|
||||
|
||||
/** Accepted requests retain their prerequisites until placement owns an instance. */
|
||||
function queuedSelections(hostId: string): QueuedSelection[] {
|
||||
// Standalone host/library readers can initialize before provisioning migrations.
|
||||
if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'").get()) return [];
|
||||
return db().prepare(`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId FROM native_creation_intents q
|
||||
if (
|
||||
!db()
|
||||
.prepare(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
return [];
|
||||
return db()
|
||||
.prepare(
|
||||
`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId FROM native_creation_intents q
|
||||
JOIN server_deployments d ON d.id=q.deployment_id
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL
|
||||
@@ -30,169 +70,337 @@ function queuedSelections(hostId: string): QueuedSelection[] {
|
||||
JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id
|
||||
WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL
|
||||
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
|
||||
AND (m.target_host_id IS NULL OR m.target_host_id=?)`).all(hostId, hostId) as QueuedSelection[];
|
||||
AND (m.target_host_id IS NULL OR m.target_host_id=?)`
|
||||
)
|
||||
.all(hostId, hostId) as QueuedSelection[];
|
||||
}
|
||||
|
||||
function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
|
||||
const latest = latestPublishedLibraries();
|
||||
const latestIds = new Set(latest.map(version => version.id));
|
||||
const latestIds = new Set(latest.map((version) => version.id));
|
||||
const queued = new Map<string, LibraryTarget>();
|
||||
const get = db().prepare(`SELECT v.id,l.game,v.version,v.size_bytes AS sizeBytes,v.manifest_sha256 AS manifestSha256
|
||||
const get = db()
|
||||
.prepare(`SELECT v.id,l.game,v.version,v.size_bytes AS sizeBytes,v.manifest_sha256 AS manifestSha256
|
||||
FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
|
||||
WHERE v.id=? AND v.state='ready' AND NOT EXISTS(SELECT 1 FROM game_file_cleanup c WHERE c.version_id=v.id)`);
|
||||
for (const selection of selections) {
|
||||
if (latestIds.has(selection.libraryId) || queued.has(selection.libraryId)) continue;
|
||||
const version = get.get(selection.libraryId) as { id: string; game: string; version: string; sizeBytes: number; manifestSha256: string } | undefined;
|
||||
if (version) queued.set(version.id, { ...version, queued: true,
|
||||
name: `${GAME_FILE_GAMES.find(game => game.id === version.game)?.name ?? version.game} · Queued version ${version.version}` });
|
||||
const version = get.get(selection.libraryId) as
|
||||
| {
|
||||
id: string;
|
||||
game: string;
|
||||
version: string;
|
||||
sizeBytes: number;
|
||||
manifestSha256: string;
|
||||
}
|
||||
| undefined;
|
||||
if (version)
|
||||
queued.set(version.id, {
|
||||
...version,
|
||||
queued: true,
|
||||
name: `${GAME_FILE_GAMES.find((game) => game.id === version.game)?.name ?? version.game} · Queued version ${version.version}`
|
||||
});
|
||||
}
|
||||
// Finish prerequisites for accepted work before downloading newer optional files.
|
||||
return [...queued.values(), ...latest];
|
||||
}
|
||||
|
||||
function latestOperation(hostId: string, kind: string, field?: string, value?: string): PreparationOperation | undefined {
|
||||
return db().prepare(`SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations
|
||||
WHERE host_id=? AND kind=? ${field ? 'AND json_extract(payload,?)=?' : ''} ORDER BY created_at DESC,rowid DESC LIMIT 1`)
|
||||
.get(hostId, kind, ...(field ? [`$.${field}`, value!] : [])) as PreparationOperation | undefined;
|
||||
function latestOperation(
|
||||
hostId: string,
|
||||
kind: string,
|
||||
field?: string,
|
||||
value?: string
|
||||
): PreparationOperation | undefined {
|
||||
return db()
|
||||
.prepare(
|
||||
`SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations
|
||||
WHERE host_id=? AND kind=? ${field ? 'AND json_extract(payload,?)=?' : ''} ORDER BY created_at DESC,rowid DESC LIMIT 1`
|
||||
)
|
||||
.get(hostId, kind, ...(field ? [`$.${field}`, value!] : [])) as
|
||||
PreparationOperation | undefined;
|
||||
}
|
||||
|
||||
function retryAt(hostId: string, target: Target): number {
|
||||
const operation = target.operation;
|
||||
if (!operation || operation.state !== 'failed') return 0;
|
||||
const kind = target.image ? 'runtime.pull' : target.versionId ? 'library.download' : 'plutonium.prepare';
|
||||
const kind = target.image
|
||||
? 'runtime.pull'
|
||||
: target.versionId
|
||||
? 'library.download'
|
||||
: 'plutonium.prepare';
|
||||
const field = target.image ? '$.image' : target.versionId ? '$.libraryVersionId' : null;
|
||||
const value = target.image ?? target.versionId;
|
||||
const match = field ? 'AND json_extract(payload,?)=?' : '';
|
||||
const bindings = [hostId, kind, ...(field ? [field, value!] : [])];
|
||||
// Count consecutive failures, including across controller restarts. Unknown
|
||||
// operations are never retried under a new ID: their journal must reconcile.
|
||||
const { count } = db().prepare(`SELECT COUNT(*) AS count FROM ec_host_operations
|
||||
const { count } = db()
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count FROM ec_host_operations
|
||||
WHERE host_id=? AND kind=? ${match} AND state='failed'
|
||||
AND rowid>COALESCE((SELECT MAX(rowid) FROM ec_host_operations
|
||||
WHERE host_id=? AND kind=? ${match} AND state='completed'),0)`)
|
||||
WHERE host_id=? AND kind=? ${match} AND state='completed'),0)`
|
||||
)
|
||||
.get(...bindings, ...bindings) as { count: number };
|
||||
return Date.parse(operation.updated_at) + Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1)));
|
||||
return (
|
||||
Date.parse(operation.updated_at) +
|
||||
Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1)))
|
||||
);
|
||||
}
|
||||
|
||||
function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null {
|
||||
if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' };
|
||||
if (host.mode !== 'active' && !host.enablingSince) return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
|
||||
if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' };
|
||||
if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
|
||||
if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' };
|
||||
if (!host.inventory.dockerRunning) return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' };
|
||||
if (!host.inventory.directoriesWritable) return { state: 'blocked', detail: 'The host cannot write its managed directories.' };
|
||||
if (host.decommission)
|
||||
return {
|
||||
state: 'waiting',
|
||||
detail: 'Automatic preparation is paused during host deletion.'
|
||||
};
|
||||
if (host.mode !== 'active' && !host.enablingSince)
|
||||
return {
|
||||
state: 'waiting',
|
||||
detail: `Automatic preparation is paused while this host is ${host.mode}.`
|
||||
};
|
||||
if (!host.enrolled)
|
||||
return {
|
||||
state: 'waiting',
|
||||
detail: 'Automatic preparation starts after agent installation.'
|
||||
};
|
||||
if (!host.connected || !host.credentialActive)
|
||||
return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
|
||||
if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION)
|
||||
return { state: 'blocked', detail: 'A compatible host agent is required.' };
|
||||
if (!host.inventory.dockerRunning)
|
||||
return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' };
|
||||
if (!host.inventory.directoriesWritable)
|
||||
return { state: 'blocked', detail: 'The host cannot write its managed directories.' };
|
||||
return null;
|
||||
}
|
||||
|
||||
function targets(host: Host): Target[] {
|
||||
const enabling = db().prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?').get(host.id) as { operation_floor: number } | undefined;
|
||||
const verifiedThisEnable = (operation: PreparationOperation | undefined) => !enabling || !!operation
|
||||
&& operation.sequence > enabling.operation_floor && operation.state === 'completed'
|
||||
&& Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at);
|
||||
const enabling = db()
|
||||
.prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?')
|
||||
.get(host.id) as { operation_floor: number } | undefined;
|
||||
const verifiedThisEnable = (operation: PreparationOperation | undefined) =>
|
||||
!enabling ||
|
||||
(!!operation &&
|
||||
operation.sequence > enabling.operation_floor &&
|
||||
operation.state === 'completed' &&
|
||||
Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at));
|
||||
const selections = queuedSelections(host.id);
|
||||
// Queued launches and migrations are not authority to pull custom images.
|
||||
const result: Target[] = [config.defaultRuntimeImage].map(image => {
|
||||
const result: Target[] = [config.defaultRuntimeImage].map((image) => {
|
||||
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
|
||||
const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
|
||||
return { image, requiredBytes: host.minimumFreeBytes,
|
||||
const runtimeReady =
|
||||
host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
|
||||
return {
|
||||
image,
|
||||
requiredBytes: host.minimumFreeBytes,
|
||||
operation,
|
||||
item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } };
|
||||
item: {
|
||||
kind: 'runtime',
|
||||
label: image,
|
||||
state: runtimeReady ? 'ready' : 'waiting',
|
||||
detail: runtimeReady
|
||||
? 'Runtime is cached.'
|
||||
: 'Runtime will be downloaded automatically.'
|
||||
}
|
||||
};
|
||||
});
|
||||
for (const version of targetLibraries(selections)) {
|
||||
const cache = getHostLibraryCache(host.id, version.id);
|
||||
const matches = cache?.manifestSha256 === version.manifestSha256;
|
||||
const operation = latestOperation(host.id, 'library.download', 'libraryVersionId', version.id);
|
||||
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes && verifiedThisEnable(operation);
|
||||
result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
|
||||
const operation = latestOperation(
|
||||
host.id,
|
||||
'library.download',
|
||||
'libraryVersionId',
|
||||
version.id
|
||||
);
|
||||
const ready =
|
||||
matches &&
|
||||
['ready', 'outdated'].includes(cache!.status) &&
|
||||
cache!.bytesDownloaded === version.sizeBytes &&
|
||||
verifiedThisEnable(operation);
|
||||
result.push({
|
||||
versionId: version.id,
|
||||
requiredBytes:
|
||||
version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
|
||||
operation,
|
||||
item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting',
|
||||
item: {
|
||||
kind: 'library',
|
||||
label: version.name,
|
||||
state: ready ? 'ready' : 'waiting',
|
||||
detail: version.queued
|
||||
? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.'
|
||||
: ready ? 'The latest game files are downloaded and verified.' : 'The latest game files will be downloaded and verified automatically.',
|
||||
bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes, ...(cache ? { cache } : {}) } });
|
||||
? ready
|
||||
? 'Game files selected by queued launches are downloaded and verified.'
|
||||
: 'Game files selected by queued launches will be downloaded and verified automatically.'
|
||||
: ready
|
||||
? 'The latest game files are downloaded and verified.'
|
||||
: 'The latest game files will be downloaded and verified automatically.',
|
||||
bytesDownloaded: cache?.bytesDownloaded ?? 0,
|
||||
bytesTotal: version.sizeBytes,
|
||||
...(cache ? { cache } : {})
|
||||
}
|
||||
});
|
||||
}
|
||||
if (host.inventory.capabilities?.includes('plutonium.cache')) {
|
||||
// Read the same production release evidence as freshRelease without
|
||||
// importing its fleet/provisioning observers into host preparation.
|
||||
const release = readSetting<PlutoniumRelease>('plutonium_release', { revision: null, checkedAt: null, error: null });
|
||||
const release = readSetting<PlutoniumRelease>('plutonium_release', {
|
||||
revision: null,
|
||||
checkedAt: null,
|
||||
error: null
|
||||
});
|
||||
const age = release.checkedAt ? Date.now() - Date.parse(release.checkedAt) : NaN;
|
||||
const revision = !release.error && age >= 0 && age < 60 * 60_000
|
||||
&& Number.isSafeInteger(release.revision) && release.revision! > 0 ? release.revision! : 0;
|
||||
const revision =
|
||||
!release.error &&
|
||||
age >= 0 &&
|
||||
age < 60 * 60_000 &&
|
||||
Number.isSafeInteger(release.revision) &&
|
||||
release.revision! > 0
|
||||
? release.revision!
|
||||
: 0;
|
||||
const cache = host.inventory.plutoniumCache;
|
||||
const ready = !!cache && cache.revision >= revision;
|
||||
result.push({ revision, requiredBytes: host.minimumFreeBytes,
|
||||
result.push({
|
||||
revision,
|
||||
requiredBytes: host.minimumFreeBytes,
|
||||
operation: latestOperation(host.id, 'plutonium.prepare'),
|
||||
item: { kind: 'plutonium', label: 'Plutonium cache (optional)', state: ready ? 'ready' : 'waiting',
|
||||
detail: ready ? `Production revision r${cache.revision} is cached for new deployments.`
|
||||
: cache ? `Production revision r${revision} will replace cached r${cache.revision} automatically.`
|
||||
: 'Plutonium files will be cached after the runtime and game files are ready.' } });
|
||||
item: {
|
||||
kind: 'plutonium',
|
||||
label: 'Plutonium cache (optional)',
|
||||
state: ready ? 'ready' : 'waiting',
|
||||
detail: ready
|
||||
? `Production revision r${cache.revision} is cached for new deployments.`
|
||||
: cache
|
||||
? `Production revision r${revision} will replace cached r${cache.revision} automatically.`
|
||||
: 'Plutonium files will be cached after the runtime and game files are ready.'
|
||||
}
|
||||
});
|
||||
}
|
||||
const blocker = hostBlocker(host);
|
||||
for (const target of result) {
|
||||
if (target.item.state === 'ready') continue;
|
||||
const operation = target.operation;
|
||||
if (operation) target.item.operationId = operation.id;
|
||||
if (blocker) { Object.assign(target.item, blocker); continue; }
|
||||
if (blocker) {
|
||||
Object.assign(target.item, blocker);
|
||||
continue;
|
||||
}
|
||||
if (operation && activeStates.includes(operation.state)) {
|
||||
target.item.state = 'preparing';
|
||||
target.item.detail = target.versionId && operation.state === 'pending' && !operation.last_delivered_at
|
||||
? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.'
|
||||
: operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
|
||||
: target.image ? 'Downloading the runtime automatically.'
|
||||
: target.revision !== undefined ? 'Downloading and verifying the optional Plutonium cache.'
|
||||
: target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
|
||||
} else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
|
||||
target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
|
||||
} else if ((target.revision !== undefined || !enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) {
|
||||
target.item.detail =
|
||||
target.versionId && operation.state === 'pending' && !operation.last_delivered_at
|
||||
? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.'
|
||||
: operation.state === 'unknown'
|
||||
? 'Waiting for the agent to reconcile interrupted preparation.'
|
||||
: target.image
|
||||
? 'Downloading the runtime automatically.'
|
||||
: target.revision !== undefined
|
||||
? 'Downloading and verifying the optional Plutonium cache.'
|
||||
: target.item.cache?.status === 'verifying'
|
||||
? 'Download complete. Verifying all game files.'
|
||||
: 'Downloading game files. Verification starts after all files arrive.';
|
||||
} else if (
|
||||
operation &&
|
||||
Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)
|
||||
) {
|
||||
target.item.state = 'preparing';
|
||||
target.item.detail = 'Waiting for a fresh host inventory after preparation.';
|
||||
} else if (
|
||||
(target.revision !== undefined ||
|
||||
!enabling ||
|
||||
(operation?.sequence ?? 0) > enabling.operation_floor) &&
|
||||
retryAt(host.id, target) > Date.now()
|
||||
) {
|
||||
target.item.state = 'blocked';
|
||||
target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
|
||||
} else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
|
||||
target.item.state = 'blocked'; target.item.detail = 'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.';
|
||||
target.item.state = 'blocked';
|
||||
target.item.detail =
|
||||
'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.';
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function hostPreparationStatus(host: Host): HostPreparation {
|
||||
const items = targets(host).map(target => target.item);
|
||||
const items = targets(host).map((target) => target.item);
|
||||
if (host.enablingSince) {
|
||||
const published = latestPublishedLibraries();
|
||||
for (const game of GAME_FILE_GAMES) if (!published.some(version => version.game === game.id)) {
|
||||
items.push({ kind: 'library', label: game.name, state: 'blocked', detail: `Upload and publish ${game.name} game files before this host can be enabled.` });
|
||||
}
|
||||
} else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
|
||||
detail: 'Upload Black Ops I or Black Ops II game files; hosts download the latest available files automatically.' });
|
||||
for (const game of GAME_FILE_GAMES)
|
||||
if (!published.some((version) => version.game === game.id)) {
|
||||
items.push({
|
||||
kind: 'library',
|
||||
label: game.name,
|
||||
state: 'blocked',
|
||||
detail: `Upload and publish ${game.name} game files before this host can be enabled.`
|
||||
});
|
||||
}
|
||||
} else if (!items.some((item) => item.kind === 'library'))
|
||||
items.push({
|
||||
kind: 'library',
|
||||
label: 'Game files',
|
||||
state: 'waiting',
|
||||
detail: 'Upload Black Ops I or Black Ops II game files; hosts download the latest available files automatically.'
|
||||
});
|
||||
// Cache health is visible, but never makes a usable host ineligible or
|
||||
// delays re-enabling: a lobby's normal updater handles cache misses.
|
||||
const required = items.filter(item => item.kind !== 'plutonium');
|
||||
const state = required.some(item => item.state === 'blocked') ? 'blocked'
|
||||
: required.some(item => item.state === 'preparing') ? 'preparing'
|
||||
: required.some(item => item.state === 'waiting') ? 'waiting' : 'ready';
|
||||
const required = items.filter((item) => item.kind !== 'plutonium');
|
||||
const state = required.some((item) => item.state === 'blocked')
|
||||
? 'blocked'
|
||||
: required.some((item) => item.state === 'preparing')
|
||||
? 'preparing'
|
||||
: required.some((item) => item.state === 'waiting')
|
||||
? 'waiting'
|
||||
: 'ready';
|
||||
return { state, items };
|
||||
}
|
||||
|
||||
export function presentHostPreparation(host: Host): Host {
|
||||
const preparation = hostPreparationStatus(host);
|
||||
if (['needs-runtime', 'needs-game-files'].includes(host.readiness) && preparation.state === 'preparing') {
|
||||
return { ...host, readiness: 'preparing', readinessReasons: preparation.items.filter(item => item.kind !== 'plutonium' && item.state !== 'ready').map(item => item.detail), preparation };
|
||||
if (
|
||||
['needs-runtime', 'needs-game-files'].includes(host.readiness) &&
|
||||
preparation.state === 'preparing'
|
||||
) {
|
||||
return {
|
||||
...host,
|
||||
readiness: 'preparing',
|
||||
readinessReasons: preparation.items
|
||||
.filter((item) => item.kind !== 'plutonium' && item.state !== 'ready')
|
||||
.map((item) => item.detail),
|
||||
preparation
|
||||
};
|
||||
}
|
||||
return { ...host, preparation };
|
||||
}
|
||||
|
||||
function supersedeUndeliveredDownloads(hostId: string): void {
|
||||
const required = new Set(targetLibraries(queuedSelections(hostId)).map(version => version.id));
|
||||
const pending = db().prepare(`SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId
|
||||
const required = new Set(
|
||||
targetLibraries(queuedSelections(hostId)).map((version) => version.id)
|
||||
);
|
||||
const pending = db()
|
||||
.prepare(
|
||||
`SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId
|
||||
FROM ec_host_operations WHERE host_id=? AND kind='library.download' AND state='pending'
|
||||
AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1`).all(hostId) as { id: string; versionId: string }[];
|
||||
AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1`
|
||||
)
|
||||
.all(hostId) as { id: string; versionId: string }[];
|
||||
const now = new Date().toISOString();
|
||||
for (const operation of pending) {
|
||||
if (required.has(operation.versionId)) continue;
|
||||
// Delivered operations may already be writing files. Only unsubmitted
|
||||
// automatic work can be replaced without consulting the agent journal.
|
||||
db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(supersededDownload, now, operation.id);
|
||||
db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)").run(operation.id, now, supersededDownload);
|
||||
db().prepare("UPDATE host_library_caches SET status='missing',error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=? AND operation_id=? AND status NOT IN ('ready','outdated')")
|
||||
db()
|
||||
.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?")
|
||||
.run(supersededDownload, now, operation.id);
|
||||
db()
|
||||
.prepare(
|
||||
"INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)"
|
||||
)
|
||||
.run(operation.id, now, supersededDownload);
|
||||
db()
|
||||
.prepare(
|
||||
"UPDATE host_library_caches SET status='missing',error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=? AND operation_id=? AND status NOT IN ('ready','outdated')"
|
||||
)
|
||||
.run(now, hostId, operation.versionId, operation.id);
|
||||
}
|
||||
}
|
||||
@@ -207,28 +415,55 @@ export function reconcileHostPreparation(hostId: string): HostPreparation {
|
||||
// Queue one required setup mutation at a time with fresh disk
|
||||
// inventory. A running optional cache yields to required work via
|
||||
// the heartbeat flag, then resumes under its original operation ID.
|
||||
const active = db().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=?
|
||||
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId);
|
||||
const finished = db().prepare(`SELECT updated_at FROM ec_host_operations WHERE host_id=?
|
||||
const active = db()
|
||||
.prepare(
|
||||
`SELECT 1 FROM ec_host_operations WHERE host_id=?
|
||||
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`
|
||||
)
|
||||
.get(hostId);
|
||||
const finished = db()
|
||||
.prepare(
|
||||
`SELECT updated_at FROM ec_host_operations WHERE host_id=?
|
||||
AND kind IN ('runtime.pull','library.download','library.import','plutonium.prepare') AND state IN ('completed','failed')
|
||||
AND (error IS NULL OR error != ?)
|
||||
ORDER BY updated_at DESC LIMIT 1`).get(hostId, supersededDownload) as { updated_at: string } | undefined;
|
||||
if (!active && (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))) {
|
||||
ORDER BY updated_at DESC LIMIT 1`
|
||||
)
|
||||
.get(hostId, supersededDownload) as { updated_at: string } | undefined;
|
||||
if (
|
||||
!active &&
|
||||
(!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))
|
||||
) {
|
||||
// A failed or oversized library must not prevent other libraries
|
||||
// from preparing. The runtime still comes first when eligible.
|
||||
const candidates = targets(host);
|
||||
const requiredReady = candidates.every(target => target.item.kind === 'plutonium' || target.item.state === 'ready');
|
||||
const target = candidates.find(target => target.item.state === 'waiting' && (target.item.kind !== 'plutonium' || requiredReady));
|
||||
if (target?.image) queueHostOperation(host.id, 'runtime.pull', { image: target.image });
|
||||
else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true });
|
||||
else if (target?.revision !== undefined && !shouldDeferPlutoniumPreparation(host.id)) queueHostOperation(host.id, 'plutonium.prepare', {
|
||||
revision: target.revision, minimumFreeBytes: host.minimumFreeBytes, automatic: true
|
||||
});
|
||||
const requiredReady = candidates.every(
|
||||
(target) => target.item.kind === 'plutonium' || target.item.state === 'ready'
|
||||
);
|
||||
const target = candidates.find(
|
||||
(target) =>
|
||||
target.item.state === 'waiting' &&
|
||||
(target.item.kind !== 'plutonium' || requiredReady)
|
||||
);
|
||||
if (target?.image)
|
||||
queueHostOperation(host.id, 'runtime.pull', { image: target.image });
|
||||
else if (target?.versionId)
|
||||
queueLibraryDownload(host.id, target.versionId, { automatic: true });
|
||||
else if (
|
||||
target?.revision !== undefined &&
|
||||
!shouldDeferPlutoniumPreparation(host.id)
|
||||
)
|
||||
queueHostOperation(host.id, 'plutonium.prepare', {
|
||||
revision: target.revision,
|
||||
minimumFreeBytes: host.minimumFreeBytes,
|
||||
automatic: true
|
||||
});
|
||||
}
|
||||
}
|
||||
const status = hostPreparationStatus(host);
|
||||
if (host.enablingSince && !hostBlocker(host) && status.state === 'ready') {
|
||||
db().prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'").run(hostId);
|
||||
db()
|
||||
.prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'")
|
||||
.run(hostId);
|
||||
db().prepare('DELETE FROM ec_host_enable_requests WHERE host_id=?').run(hostId);
|
||||
scheduleAvailabilityRefresh();
|
||||
}
|
||||
|
||||
@@ -2,11 +2,15 @@ import { readFile } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
/** adapter-node copies public assets to build/client; Vite serves static in development. */
|
||||
export async function agentReleaseAsset(file: 'install.sh' | 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json'): Promise<string> {
|
||||
export async function agentReleaseAsset(
|
||||
file: 'install.sh' | 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json'
|
||||
): Promise<string> {
|
||||
// A prior production build must never shadow a freshly packaged development release.
|
||||
if (process.env.NODE_ENV === 'development') return readFile(resolve('static/agent', file), 'utf8');
|
||||
try { return await readFile(resolve('build/client/agent', file), 'utf8'); }
|
||||
catch (cause) {
|
||||
if (process.env.NODE_ENV === 'development')
|
||||
return readFile(resolve('static/agent', file), 'utf8');
|
||||
try {
|
||||
return await readFile(resolve('build/client/agent', file), 'utf8');
|
||||
} catch (cause) {
|
||||
if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause;
|
||||
return readFile(resolve('static/agent', file), 'utf8');
|
||||
}
|
||||
|
||||
@@ -7,8 +7,9 @@ const requests = new AsyncLocalStorage<{ inventories: InventoryCache | null }>()
|
||||
export function withHostRequestContext<T>(work: () => T | Promise<T>): Promise<T> {
|
||||
const context = { inventories: new Map() as InventoryCache | null };
|
||||
return requests.run(context, async () => {
|
||||
try { return await work(); }
|
||||
finally {
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
// Detached work can inherit this context. Release request data and
|
||||
// stop caching when the response is ready, even if that work continues.
|
||||
context.inventories = null;
|
||||
|
||||
@@ -1,16 +1,29 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js';
|
||||
import {
|
||||
ensureFileBlobSchema,
|
||||
migrationRows,
|
||||
packFileJson,
|
||||
referenceBlobs
|
||||
} from '../db/file-blobs.js';
|
||||
import { retentionClocks } from '../db/retention-clocks.js';
|
||||
import { installHistoryCounts } from './history-counts.js';
|
||||
import { migrationStep } from '../db/connection.js';
|
||||
import type Database from 'better-sqlite3';
|
||||
import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
|
||||
import {
|
||||
successfulObservationSql,
|
||||
routineObservationSql,
|
||||
operationTargetSql,
|
||||
importantOperationSql
|
||||
} from './observations.js';
|
||||
|
||||
const initialized = new WeakSet<Database.Database>();
|
||||
export function ensureHostSchema(handle: Database.Database): void {
|
||||
if (initialized.has(handle)) return;
|
||||
// Background readers open only after controller migrations have completed.
|
||||
if (handle.readonly) { initialized.add(handle); return; }
|
||||
if (handle.readonly) {
|
||||
initialized.add(handle);
|
||||
return;
|
||||
}
|
||||
migrationStep('host schema and saved command files', () => migrateHostSchema(handle));
|
||||
initialized.add(handle);
|
||||
}
|
||||
@@ -105,26 +118,39 @@ function migrateHostSchema(handle: Database.Database): void {
|
||||
CREATE INDEX IF NOT EXISTS idx_ec_host_credentials_host ON ec_host_credentials(host_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ec_host_enrollments_host ON ec_host_enrollments(host_id);
|
||||
`);
|
||||
const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
|
||||
if (!columns.some(column => column.name === 'history_expired_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN history_expired_at TEXT');
|
||||
if (!columns.some(column => column.name === 'request_hash')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN request_hash TEXT');
|
||||
if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
|
||||
if (!columns.some(column => column.name === 'observation_success')) {
|
||||
const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as {
|
||||
name: string;
|
||||
}[];
|
||||
if (!columns.some((column) => column.name === 'history_expired_at'))
|
||||
handle.exec('ALTER TABLE ec_host_operations ADD COLUMN history_expired_at TEXT');
|
||||
if (!columns.some((column) => column.name === 'request_hash'))
|
||||
handle.exec('ALTER TABLE ec_host_operations ADD COLUMN request_hash TEXT');
|
||||
if (!columns.some((column) => column.name === 'last_delivered_at'))
|
||||
handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
|
||||
if (!columns.some((column) => column.name === 'observation_success')) {
|
||||
handle.transaction(() => {
|
||||
handle.exec('ALTER TABLE ec_host_operations ADD COLUMN observation_success INTEGER NOT NULL DEFAULT 0');
|
||||
handle.exec(`UPDATE ec_host_operations AS o SET observation_success=1 WHERE ${successfulObservationSql}`);
|
||||
handle.exec(
|
||||
'ALTER TABLE ec_host_operations ADD COLUMN observation_success INTEGER NOT NULL DEFAULT 0'
|
||||
);
|
||||
handle.exec(
|
||||
`UPDATE ec_host_operations AS o SET observation_success=1 WHERE ${successfulObservationSql}`
|
||||
);
|
||||
})();
|
||||
}
|
||||
handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_observation_retention ON ec_host_operations(updated_at)
|
||||
WHERE observation_success=1 AND state='completed'`);
|
||||
if (!columns.some(column => column.name === 'routine')) {
|
||||
if (!columns.some((column) => column.name === 'routine')) {
|
||||
handle.transaction(() => {
|
||||
handle.exec(`ALTER TABLE ec_host_operations ADD COLUMN routine INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE ec_host_operations ADD COLUMN server_name TEXT;
|
||||
ALTER TABLE ec_host_operations ADD COLUMN target_path TEXT;
|
||||
ALTER TABLE ec_host_operations ADD COLUMN source TEXT;`);
|
||||
const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as { name: string }[];
|
||||
const nativeInstances = ['id', 'server'].every(name => nativeColumns.some(column => column.name === name));
|
||||
const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as {
|
||||
name: string;
|
||||
}[];
|
||||
const nativeInstances = ['id', 'server'].every((name) =>
|
||||
nativeColumns.some((column) => column.name === name)
|
||||
);
|
||||
handle.exec(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
|
||||
target_path=${operationTargetSql}, server_name=${nativeInstances ? '(SELECT n.server FROM native_instances n WHERE n.id=o.instance_id)' : 'NULL'}`);
|
||||
})();
|
||||
@@ -135,59 +161,99 @@ function migrateHostSchema(handle: Database.Database): void {
|
||||
CREATE INDEX IF NOT EXISTS ec_host_operations_important_history ON ec_host_operations(created_at)
|
||||
WHERE ${importantOperationSql.replaceAll('o.', '')}`);
|
||||
handle.exec('CREATE TABLE IF NOT EXISTS ec_host_schema_migrations (name TEXT PRIMARY KEY)');
|
||||
if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('staged-operation-payloads')) {
|
||||
if (
|
||||
!handle
|
||||
.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?')
|
||||
.get('staged-operation-payloads')
|
||||
) {
|
||||
handle.transaction(() => {
|
||||
handle.exec(`INSERT OR IGNORE INTO ec_host_operation_payloads(operation_id,payload)
|
||||
SELECT id,payload FROM ec_host_operations WHERE kind='files.stage' AND json_type(payload,'$.files')='array';
|
||||
UPDATE ec_host_operations SET payload=json_object('stageId',json_extract(payload,'$.stageId'))
|
||||
WHERE kind='files.stage' AND json_type(payload,'$.files')='array';`);
|
||||
handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('staged-operation-payloads');
|
||||
handle
|
||||
.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)')
|
||||
.run('staged-operation-payloads');
|
||||
})();
|
||||
}
|
||||
if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('completed-status-batches')) {
|
||||
if (
|
||||
!handle
|
||||
.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?')
|
||||
.get('completed-status-batches')
|
||||
) {
|
||||
handle.transaction(() => {
|
||||
// Reclassify existing completed batches without discarding their per-server replies.
|
||||
handle.exec(`UPDATE ec_host_operations AS o SET observation_success=1
|
||||
WHERE o.kind='rcon.status-batch' AND o.observation_success=0 AND ${successfulObservationSql}`);
|
||||
handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('completed-status-batches');
|
||||
handle
|
||||
.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)')
|
||||
.run('completed-status-batches');
|
||||
})();
|
||||
}
|
||||
const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
|
||||
if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
|
||||
const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[];
|
||||
if (!allowanceColumns.some(column => column.name === 'usage_margin')) {
|
||||
if (!hostColumns.some((column) => column.name === 'deleted_at'))
|
||||
handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
|
||||
const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as {
|
||||
name: string;
|
||||
}[];
|
||||
if (!allowanceColumns.some((column) => column.name === 'usage_margin')) {
|
||||
// Existing caches include 25% padding. Preserve learned evidence while
|
||||
// removing that padding exactly once, including dormant startup models.
|
||||
handle.transaction(() => {
|
||||
handle.exec('ALTER TABLE ec_resource_allowances ADD COLUMN usage_margin REAL NOT NULL DEFAULT 1');
|
||||
handle.exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)');
|
||||
handle.exec(
|
||||
'ALTER TABLE ec_resource_allowances ADD COLUMN usage_margin REAL NOT NULL DEFAULT 1'
|
||||
);
|
||||
handle.exec(
|
||||
'UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)'
|
||||
);
|
||||
})();
|
||||
}
|
||||
if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('operation-file-blobs')) {
|
||||
if (
|
||||
!handle
|
||||
.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?')
|
||||
.get('operation-file-blobs')
|
||||
) {
|
||||
handle.transaction(() => {
|
||||
for (const row of migrationRows<{ id: string; payload: string; result: string | null }>(handle,
|
||||
for (const row of migrationRows<{ id: string; payload: string; result: string | null }>(
|
||||
handle,
|
||||
`SELECT o.rowid AS rid,o.id,COALESCE(p.payload,o.payload) payload,o.result
|
||||
FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id
|
||||
WHERE o.rowid>? AND o.kind IN ('files.stage','instance.create','instance.rebuild','instance.restart','instance.start','files.read')
|
||||
ORDER BY o.rowid LIMIT 1`)) {
|
||||
ORDER BY o.rowid LIMIT 1`
|
||||
)) {
|
||||
const packed = packFileJson(JSON.parse(row.payload), handle);
|
||||
handle.prepare('UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?) WHERE id=?')
|
||||
handle
|
||||
.prepare(
|
||||
'UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?) WHERE id=?'
|
||||
)
|
||||
.run(createHash('sha256').update(row.payload).digest('hex'), row.id);
|
||||
if (packed.hashes.size) {
|
||||
handle.prepare(`INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)
|
||||
ON CONFLICT(operation_id) DO UPDATE SET payload=excluded.payload`).run(row.id, JSON.stringify(packed.value));
|
||||
handle.prepare("UPDATE ec_host_operations SET payload=json_remove(payload,'$.files') WHERE id=?").run(row.id);
|
||||
handle
|
||||
.prepare(
|
||||
`INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)
|
||||
ON CONFLICT(operation_id) DO UPDATE SET payload=excluded.payload`
|
||||
)
|
||||
.run(row.id, JSON.stringify(packed.value));
|
||||
handle
|
||||
.prepare(
|
||||
"UPDATE ec_host_operations SET payload=json_remove(payload,'$.files') WHERE id=?"
|
||||
)
|
||||
.run(row.id);
|
||||
referenceBlobs('operation', row.id, packed.hashes, handle);
|
||||
}
|
||||
if (row.result) {
|
||||
const result = packFileJson(JSON.parse(row.result), handle);
|
||||
if (result.hashes.size) {
|
||||
handle.prepare('UPDATE ec_host_operations SET result=? WHERE id=?').run(JSON.stringify(result.value), row.id);
|
||||
handle
|
||||
.prepare('UPDATE ec_host_operations SET result=? WHERE id=?')
|
||||
.run(JSON.stringify(result.value), row.id);
|
||||
referenceBlobs('operation-result', row.id, result.hashes, handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('operation-file-blobs');
|
||||
handle
|
||||
.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)')
|
||||
.run('operation-file-blobs');
|
||||
})();
|
||||
}
|
||||
handle.exec(`CREATE TRIGGER IF NOT EXISTS release_operation_blobs AFTER DELETE ON ec_host_operations BEGIN
|
||||
|
||||
Reference in New Issue
Block a user