1724 lines
75 KiB
TypeScript
1724 lines
75 KiB
TypeScript
import {
|
|
assertPublicFilePayload,
|
|
BANK_CONFIG_FILE,
|
|
PrivateFileOperationError,
|
|
isPrivateConfigPath
|
|
} from '../fleet/private-config.js';
|
|
import { packFileJson, unpackFileJson, referenceBlobs } from '../db/file-blobs.js';
|
|
import { pruneHistory } from '../retention.js';
|
|
import { recordAgentReceipt } from '../provisioning/agent-events.js';
|
|
import { EventEmitter } from 'node:events';
|
|
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
|
|
import { isIP } from 'node:net';
|
|
import { db } from '../db/index.js';
|
|
import {
|
|
successfulObservation,
|
|
routineObservationSql,
|
|
importantOperationSql,
|
|
operationTargetSql,
|
|
safeOperationTarget
|
|
} from './observations.js';
|
|
import { currentOperationSource } from './operation-context.js';
|
|
import { logResultMetadata } from './log-results.js';
|
|
import { parseHostInventory } from './request-context.js';
|
|
import { fleetSettings } from '../settings.js';
|
|
import { recordInstanceResources, validateInstanceResources } from './calibration.js';
|
|
import { validateTemperatures } from '../../host-temperatures.js';
|
|
import { recordTemperatures } from './temperatures.js';
|
|
import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
|
|
import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js';
|
|
import type {
|
|
Host,
|
|
HostDecommission,
|
|
HostDeletion,
|
|
HostInventory,
|
|
HostMode,
|
|
HostOperation,
|
|
HostOperationFilter,
|
|
HostOperationHistoryPage,
|
|
HostOperationScope,
|
|
HostOperationState
|
|
} from '../../hosts.js';
|
|
|
|
export const AGENT_PROTOCOL_VERSION = 1;
|
|
export const HEARTBEAT_MAX_AGE_MS = 90_000;
|
|
const now = () => new Date().toISOString();
|
|
const workEvents = new EventEmitter();
|
|
// Hold reply bodies only while a reader is awaiting this operation. No TTL cache or disk copy.
|
|
const logReaders = new Map<string, Set<{ received: boolean; result?: unknown }>>();
|
|
workEvents.setMaxListeners(0);
|
|
export function waitForHostWork(
|
|
hostId: string,
|
|
signal: AbortSignal,
|
|
timeoutMs = 20_000
|
|
): Promise<void> {
|
|
return new Promise((resolve) => {
|
|
const done = () => {
|
|
clearTimeout(timer);
|
|
workEvents.off(hostId, done);
|
|
signal.removeEventListener('abort', done);
|
|
resolve();
|
|
};
|
|
const timer = setTimeout(done, timeoutMs);
|
|
workEvents.on(hostId, done);
|
|
signal.addEventListener('abort', done, { once: true });
|
|
if (signal.aborted) done();
|
|
});
|
|
}
|
|
const digest = (token: string) => createHash('sha256').update(token).digest('hex');
|
|
function database() {
|
|
return db();
|
|
}
|
|
type HostRow = {
|
|
id: string;
|
|
name: string;
|
|
hostname: string;
|
|
mode: HostMode;
|
|
port_min: number;
|
|
port_max: number;
|
|
bind_address: string;
|
|
public_address: string;
|
|
minimum_free_bytes: number;
|
|
created_at: string;
|
|
enrolled_at: string | null;
|
|
last_heartbeat: string | null;
|
|
inventory: string;
|
|
credential_active: number;
|
|
};
|
|
const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`;
|
|
type DecommissionRow = {
|
|
host_id: string;
|
|
operation_id: string;
|
|
token_hash: string;
|
|
stage: HostDecommission['stage'];
|
|
requested_at: string;
|
|
updated_at: string;
|
|
cleaned_at: string | null;
|
|
error: string | null;
|
|
};
|
|
function decommissionRow(hostId: string): DecommissionRow | undefined {
|
|
return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as
|
|
DecommissionRow | undefined;
|
|
}
|
|
function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null {
|
|
return row
|
|
? {
|
|
operationId: row.operation_id,
|
|
stage: row.stage,
|
|
requestedAt: row.requested_at,
|
|
updatedAt: row.updated_at,
|
|
error: row.error,
|
|
retryable:
|
|
row.stage === 'failed' &&
|
|
getHostOperationStatus(row.operation_id)?.state === 'failed'
|
|
}
|
|
: null;
|
|
}
|
|
function hostFromRow(row: HostRow): Host {
|
|
const enabling = database()
|
|
.prepare('SELECT requested_at FROM ec_host_enable_requests WHERE host_id=?')
|
|
.get(row.id) as { requested_at: string } | undefined;
|
|
const inventory = parseHostInventory(row.id, row.inventory);
|
|
const connected =
|
|
!!row.credential_active &&
|
|
!!row.last_heartbeat &&
|
|
Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
|
|
const reasons: string[] = [];
|
|
if (!row.enrolled_at) reasons.push('Agent has not enrolled.');
|
|
else if (!row.credential_active) reasons.push('Agent credential has been revoked.');
|
|
if (!connected) reasons.push('Agent heartbeat is unavailable or stale.');
|
|
if (connected)
|
|
for (const lane of inventory.agentLanes ?? []) {
|
|
const state = hostAgentLaneState(lane);
|
|
if (state === 'stopped' || state === 'stalled')
|
|
reasons.push(
|
|
`Agent ${hostAgentLaneLabel(lane.name).toLowerCase()} worker is ${state}. Check the host agent service.`
|
|
);
|
|
}
|
|
if (inventory.protocolVersion !== AGENT_PROTOCOL_VERSION)
|
|
reasons.push('A compatible agent protocol is required.');
|
|
if (!inventory.dockerRunning) reasons.push('Docker is not ready.');
|
|
if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.');
|
|
if (
|
|
typeof inventory.storageAvailable !== 'number' ||
|
|
inventory.storageAvailable < row.minimum_free_bytes
|
|
)
|
|
reasons.push('Insufficient available game storage.');
|
|
const infrastructureReady = reasons.length === 0;
|
|
const runtimeReady = !!inventory.preparedRuntimeImages?.length;
|
|
const libraryReady = !!inventory.libraries?.some((library) => library.state === 'ready');
|
|
if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.');
|
|
if (!libraryReady) reasons.push('No verified game-file library is available yet.');
|
|
if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
|
|
if (row.mode === 'disabled')
|
|
reasons.push(
|
|
enabling
|
|
? 'Host setup must refresh the runtime and verify both BO1 and BO2 game files before enabling.'
|
|
: 'Host is disabled.'
|
|
);
|
|
const decommission = presentDecommission(decommissionRow(row.id));
|
|
if (decommission)
|
|
reasons.push(
|
|
'Host deletion is in progress; waiting for confirmed cleanup and agent removal.'
|
|
);
|
|
return {
|
|
id: row.id,
|
|
name: row.name,
|
|
hostname: row.hostname,
|
|
mode: row.mode,
|
|
portMin: row.port_min,
|
|
portMax: row.port_max,
|
|
bindAddress: row.bind_address,
|
|
publicAddress: row.public_address,
|
|
minimumFreeBytes: row.minimum_free_bytes,
|
|
createdAt: row.created_at,
|
|
lastHeartbeat: row.last_heartbeat,
|
|
connected,
|
|
connectionState: connected ? 'online' : 'offline',
|
|
enablingSince: enabling?.requested_at ?? null,
|
|
readiness: decommission
|
|
? 'deleting'
|
|
: enabling
|
|
? 'preparing'
|
|
: row.mode === 'draining'
|
|
? 'draining'
|
|
: row.mode === 'disabled'
|
|
? 'disabled'
|
|
: !row.enrolled_at
|
|
? 'preparing'
|
|
: !infrastructureReady
|
|
? 'degraded'
|
|
: !runtimeReady
|
|
? 'needs-runtime'
|
|
: !libraryReady
|
|
? 'needs-game-files'
|
|
: 'ready',
|
|
readinessReasons: reasons,
|
|
inventory,
|
|
enrolled: !!row.enrolled_at,
|
|
credentialActive: !!row.credential_active,
|
|
decommission
|
|
};
|
|
}
|
|
export function listHosts(): Host[] {
|
|
return (
|
|
database()
|
|
.prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id')
|
|
.all() as HostRow[]
|
|
).map(hostFromRow);
|
|
}
|
|
export function getHost(id: string): Host | null {
|
|
const row = database()
|
|
.prepare(hostSql + ' WHERE h.id = ? AND h.deleted_at IS NULL')
|
|
.get(id) as HostRow | undefined;
|
|
return row ? hostFromRow(row) : null;
|
|
}
|
|
function requireHost(id: string): Host {
|
|
const host = getHost(id);
|
|
if (!host) throw new Error('Host was not found.');
|
|
return host;
|
|
}
|
|
|
|
export class HostDeletionError extends Error {
|
|
constructor(
|
|
message: string,
|
|
readonly status: 400 | 401 | 404 | 409,
|
|
readonly blockers: string[] = []
|
|
) {
|
|
super(message);
|
|
}
|
|
}
|
|
|
|
/** Controller records can be removed only after all remote instance ownership is resolved. */
|
|
export function hostDeletionStatus(id: string): HostDeletion {
|
|
const host = getHost(id);
|
|
if (!host) throw new HostDeletionError('Host not found.', 404);
|
|
if (host.decommission && !host.decommission.retryable) return { allowed: true, blockers: [] };
|
|
const handle = database();
|
|
const blockers: string[] = [];
|
|
if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission'))
|
|
blockers.push(
|
|
'Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.'
|
|
);
|
|
if (host.enrolled && !host.credentialActive)
|
|
blockers.push(
|
|
'Re-enroll this host so its agent can receive and complete the cleanup request.'
|
|
);
|
|
const instances = handle
|
|
.prepare(
|
|
'SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server'
|
|
)
|
|
.all(id) as { server: string }[];
|
|
if (instances.length) {
|
|
const names =
|
|
instances
|
|
.slice(0, 5)
|
|
.map((instance) => instance.server)
|
|
.join(', ') + (instances.length > 5 ? ', …' : '');
|
|
blockers.push(
|
|
`Remove the ${instances.length} server instance${instances.length === 1 ? '' : 's'} assigned to this host first (${names}). Queued and failed deployments also require removal.`
|
|
);
|
|
}
|
|
const pinned = handle
|
|
.prepare(
|
|
`SELECT d.server FROM server_deployments d
|
|
JOIN native_creation_intents q ON q.deployment_id=d.id
|
|
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
|
WHERE d.action='create' AND d.server_deleted_at IS NULL
|
|
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND n.deployment_id IS NULL
|
|
AND json_extract(q.selection,'$.hostId')=? AND COALESCE(json_extract(q.selection,'$.automatic'),0)!=1
|
|
ORDER BY d.server LIMIT 6`
|
|
)
|
|
.all(id) as { server: string }[];
|
|
if (pinned.length) {
|
|
const names =
|
|
pinned
|
|
.slice(0, 5)
|
|
.map((job) => job.server)
|
|
.join(', ') + (pinned.length > 5 ? ', …' : '');
|
|
blockers.push(
|
|
`Cancel the queued server launches pinned to this host before deleting it (${names}).`
|
|
);
|
|
}
|
|
const ports = handle
|
|
.prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE host_id=?')
|
|
.get(id) as { count: number };
|
|
if (ports.count && !instances.length)
|
|
blockers.push(
|
|
`This host still has ${ports.count} reserved game port${ports.count === 1 ? '' : 's'}. Finish or reconcile the corresponding server removals first.`
|
|
);
|
|
// A pending command may have reached the agent before its acknowledgement.
|
|
// Only commands that have never been delivered are safe to cancel locally.
|
|
const operations = handle
|
|
.prepare(
|
|
`SELECT kind,state FROM ec_host_operations WHERE host_id=?
|
|
AND (state IN ('accepted','running','unknown') OR (state='pending' AND last_delivered_at IS NOT NULL))
|
|
ORDER BY created_at`
|
|
)
|
|
.all(id) as { kind: string; state: string }[];
|
|
if (operations.length) {
|
|
const work =
|
|
operations
|
|
.slice(0, 5)
|
|
.map((operation) => `${operation.kind}: ${operation.state}`)
|
|
.join(', ') + (operations.length > 5 ? ', …' : '');
|
|
blockers.push(
|
|
`${operations.length} delivered operation${operations.length === 1 ? ' is' : 's are'} unresolved (${work}). Let the agent reconnect and finish or reconcile this work before deleting the host.`
|
|
);
|
|
}
|
|
return { allowed: blockers.length === 0, blockers };
|
|
}
|
|
|
|
/** Keep the host visible and authenticated until its independent cleanup worker proves removal. */
|
|
export function deleteHost(id: string): { deleted: boolean; operationId?: string } {
|
|
const handle = database();
|
|
return handle
|
|
.transaction(() => {
|
|
const host = getHost(id);
|
|
if (!host) throw new HostDeletionError('Host not found.', 404);
|
|
if (host.decommission && !host.decommission.retryable)
|
|
return { deleted: false, operationId: host.decommission.operationId };
|
|
const status = hostDeletionStatus(id);
|
|
if (!status.allowed)
|
|
throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers);
|
|
const timestamp = now();
|
|
const reason =
|
|
'Host deletion requested before this operation was delivered to the agent.';
|
|
handle
|
|
.prepare(
|
|
`INSERT INTO ec_host_operation_events(operation_id,state,created_at,error)
|
|
SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL`
|
|
)
|
|
.run(timestamp, reason, id);
|
|
handle
|
|
.prepare(
|
|
"UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL"
|
|
)
|
|
.run(reason, timestamp, id);
|
|
handle
|
|
.prepare(
|
|
'UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL'
|
|
)
|
|
.run(timestamp, id);
|
|
if (!host.enrolled) {
|
|
handle
|
|
.prepare(
|
|
"UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL"
|
|
)
|
|
.run(timestamp, id);
|
|
return { deleted: true };
|
|
}
|
|
const operationId = randomUUID();
|
|
const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
|
|
const payload = {
|
|
hostId: id,
|
|
operationId,
|
|
receiptToken: token,
|
|
receiptUrl: `/api/agent/decommission/${operationId}`
|
|
};
|
|
handle
|
|
.prepare(
|
|
"INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,source) VALUES(?,?,'host.decommission',?,'pending',?,?,?)"
|
|
)
|
|
.run(
|
|
operationId,
|
|
id,
|
|
JSON.stringify(payload),
|
|
timestamp,
|
|
timestamp,
|
|
(currentOperationSource() ?? 'Controller').slice(0, 200)
|
|
);
|
|
handle
|
|
.prepare(
|
|
"INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)"
|
|
)
|
|
.run(operationId, timestamp);
|
|
handle
|
|
.prepare(
|
|
`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
|
|
ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`
|
|
)
|
|
.run(id, operationId, digest(token), timestamp, timestamp);
|
|
handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
|
|
queueMicrotask(() => workEvents.emit(id));
|
|
return { deleted: false, operationId };
|
|
})
|
|
.immediate();
|
|
}
|
|
|
|
export function receiveDecommissionReceipt(
|
|
operationId: string,
|
|
token: string,
|
|
input: Record<string, unknown>
|
|
): { acknowledged: true } {
|
|
if (!token || token.length > 200)
|
|
throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
|
|
return database()
|
|
.transaction(() => {
|
|
const row = database()
|
|
.prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?')
|
|
.get(operationId) as DecommissionRow | undefined;
|
|
if (
|
|
!row ||
|
|
!timingSafeEqual(
|
|
Buffer.from(row.token_hash, 'hex'),
|
|
Buffer.from(digest(token), 'hex')
|
|
)
|
|
)
|
|
throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
|
|
if (!['cleaned', 'completed', 'failed'].includes(input.stage as string))
|
|
throw new HostDeletionError('Invalid cleanup receipt stage.', 400);
|
|
if (row.stage === 'completed') return { acknowledged: true as const };
|
|
const operation = getHostOperationStatus(operationId)!;
|
|
if (!['accepted', 'running', 'unknown'].includes(operation.state))
|
|
throw new HostDeletionError('Cleanup has not been accepted by the host.', 409);
|
|
const timestamp = now();
|
|
if (input.stage === 'failed') {
|
|
database()
|
|
.prepare(
|
|
"UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?"
|
|
)
|
|
.run(
|
|
typeof input.error === 'string'
|
|
? input.error.slice(0, 2000)
|
|
: 'Host cleanup needs attention. The cleanup worker will retry.',
|
|
timestamp,
|
|
row.host_id
|
|
);
|
|
return { acknowledged: true as const };
|
|
}
|
|
if (input.stage === 'cleaned') {
|
|
database()
|
|
.prepare(
|
|
"UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?"
|
|
)
|
|
.run(timestamp, timestamp, row.host_id);
|
|
return { acknowledged: true as const };
|
|
}
|
|
if (!row.cleaned_at)
|
|
throw new HostDeletionError(
|
|
'The host must confirm file cleanup before confirming agent removal.',
|
|
409
|
|
);
|
|
database()
|
|
.prepare(
|
|
"UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?"
|
|
)
|
|
.run(timestamp, row.host_id);
|
|
database()
|
|
.prepare(
|
|
"UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?"
|
|
)
|
|
.run(
|
|
JSON.stringify({ filesRemoved: true, agentRemoved: true }),
|
|
timestamp,
|
|
operationId
|
|
);
|
|
database()
|
|
.prepare(
|
|
"INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)"
|
|
)
|
|
.run(operationId, timestamp);
|
|
database()
|
|
.prepare(
|
|
'UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL'
|
|
)
|
|
.run(timestamp, row.host_id);
|
|
database()
|
|
.prepare(
|
|
'UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL'
|
|
)
|
|
.run(timestamp, row.host_id);
|
|
if (
|
|
database()
|
|
.prepare(
|
|
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'"
|
|
)
|
|
.get()
|
|
) {
|
|
database()
|
|
.prepare(
|
|
"UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?"
|
|
)
|
|
.run(timestamp, row.host_id);
|
|
}
|
|
database()
|
|
.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?")
|
|
.run(timestamp, row.host_id);
|
|
return { acknowledged: true as const };
|
|
})
|
|
.immediate();
|
|
}
|
|
function textField(value: unknown, label: string, max = 200): string {
|
|
if (
|
|
typeof value !== 'string' ||
|
|
!value.trim() ||
|
|
value.length > max ||
|
|
// eslint-disable-next-line no-control-regex -- Reject control characters in host text fields.
|
|
/[\x00-\x1f]/.test(value)
|
|
)
|
|
throw new Error(`${label} must be a nonempty string up to ${max} characters.`);
|
|
return value.trim();
|
|
}
|
|
function port(value: unknown): number {
|
|
if (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 65535)
|
|
throw new Error('Ports must be whole numbers between 1 and 65535.');
|
|
return value;
|
|
}
|
|
export function createHost(input: Record<string, unknown>): Host {
|
|
const id = randomUUID();
|
|
const name = textField(input.name, 'Host name');
|
|
const min = port(input.portMin ?? 28960);
|
|
const max = port(input.portMax ?? 28980);
|
|
if (max < min) throw new Error('The last game port must not precede the first.');
|
|
const bind =
|
|
input.bindAddress === undefined ? '0.0.0.0' : textField(input.bindAddress, 'Bind address');
|
|
if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.');
|
|
database()
|
|
.prepare(
|
|
'INSERT INTO ec_hosts(id,name,port_min,port_max,bind_address,created_at) VALUES(?,?,?,?,?,?)'
|
|
)
|
|
.run(id, name, min, max, bind, now());
|
|
scheduleAvailabilityRefresh();
|
|
return requireHost(id);
|
|
}
|
|
export function updateHost(id: string, input: Record<string, unknown>): Host {
|
|
const host = requireHost(id);
|
|
if (host.decommission)
|
|
throw new Error('Host cleanup is in progress; its settings cannot change.');
|
|
const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
|
|
let mode = input.mode ?? host.mode;
|
|
if (!['active', 'draining', 'disabled'].includes(mode as string))
|
|
throw new Error('Host mode must be active, draining, or disabled.');
|
|
const min = port(input.portMin ?? host.portMin);
|
|
const max = port(input.portMax ?? host.portMax);
|
|
if (max < min) throw new Error('The last game port must not precede the first.');
|
|
const bind =
|
|
input.bindAddress === undefined
|
|
? host.bindAddress
|
|
: textField(input.bindAddress, 'Bind address');
|
|
if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.');
|
|
const publicAddress =
|
|
input.publicAddress === undefined
|
|
? host.publicAddress
|
|
: input.publicAddress === ''
|
|
? ''
|
|
: textField(input.publicAddress, 'Public address');
|
|
const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes;
|
|
if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0)
|
|
throw new Error('Minimum free storage must be a nonnegative byte count.');
|
|
database().transaction(() => {
|
|
if (input.mode === 'active' && host.mode === 'draining')
|
|
throw new Error('Wait for this host to finish draining before enabling it.');
|
|
if (input.mode === 'active' && host.mode === 'disabled') {
|
|
database()
|
|
.prepare(
|
|
'INSERT OR IGNORE INTO ec_host_enable_requests(host_id,requested_at,operation_floor) VALUES(?,?,(SELECT COALESCE(MAX(rowid),0) FROM ec_host_operations))'
|
|
)
|
|
.run(id, now());
|
|
mode = 'disabled';
|
|
}
|
|
if (input.mode === 'draining') {
|
|
if (host.enablingSince)
|
|
throw new Error('Wait for automatic host setup to finish before draining.');
|
|
const incoming = database()
|
|
.prepare(
|
|
`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
|
|
WHERE m.target_host_id=? AND d.state NOT IN ('done','warning') AND d.server_deleted_at IS NULL LIMIT 1`
|
|
)
|
|
.get(id);
|
|
if (incoming)
|
|
throw new Error(
|
|
'Wait for incoming instance migrations to finish before draining this host.'
|
|
);
|
|
}
|
|
database()
|
|
.prepare(
|
|
'UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?'
|
|
)
|
|
.run(name, mode, min, max, bind, publicAddress, minimum, id);
|
|
})();
|
|
scheduleAvailabilityRefresh();
|
|
return requireHost(id);
|
|
}
|
|
|
|
export function createEnrollment(
|
|
hostId: string,
|
|
ttlMs = 15 * 60_000
|
|
): { id: string; hostId: string; token: string; expiresAt: string } {
|
|
const host = requireHost(hostId);
|
|
if (host.decommission && !host.decommission.retryable)
|
|
throw new Error('Host cleanup is in progress; enrollment is unavailable.');
|
|
if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000)
|
|
throw new Error('Enrollment lifetime must be at most one hour.');
|
|
const token = 'ec_enroll_' + randomBytes(32).toString('base64url');
|
|
const id = randomUUID();
|
|
const expiresAt = new Date(Date.now() + ttlMs).toISOString();
|
|
database().transaction(() => {
|
|
database()
|
|
.prepare(
|
|
'UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL'
|
|
)
|
|
.run(now(), hostId);
|
|
database()
|
|
.prepare(
|
|
'INSERT INTO ec_host_enrollments(id,host_id,token_hash,created_at,expires_at) VALUES(?,?,?,?,?)'
|
|
)
|
|
.run(id, hostId, digest(token), now(), expiresAt);
|
|
})();
|
|
return { id, hostId, token, expiresAt };
|
|
}
|
|
function issueCredential(
|
|
hostId: string,
|
|
credential = 'ec_host_' + randomBytes(32).toString('base64url')
|
|
): string {
|
|
database()
|
|
.prepare(
|
|
'INSERT INTO ec_host_credentials(id,host_id,token_hash,created_at) VALUES(?,?,?,?)'
|
|
)
|
|
.run(randomUUID(), hostId, digest(credential), now());
|
|
return credential;
|
|
}
|
|
export function validateInventory(value: unknown): HostInventory {
|
|
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
throw new Error('Host inventory must be an object.');
|
|
const inventory = value as HostInventory;
|
|
if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.');
|
|
if (inventory.temperatures !== undefined)
|
|
inventory.temperatures = validateTemperatures(inventory.temperatures);
|
|
for (const key of ['hostname', 'agentVersion', 'os', 'architecture', 'cpu', 'dockerVersion']) {
|
|
if (
|
|
inventory[key] !== undefined &&
|
|
(typeof inventory[key] !== 'string' || String(inventory[key]).length > 1000)
|
|
)
|
|
throw new Error(`Invalid inventory ${key}.`);
|
|
}
|
|
for (const key of [
|
|
'cpuCount',
|
|
'memoryTotal',
|
|
'memoryAvailable',
|
|
'storageTotal',
|
|
'storageAvailable',
|
|
'protocolVersion'
|
|
]) {
|
|
if (
|
|
inventory[key] !== undefined &&
|
|
(typeof inventory[key] !== 'number' ||
|
|
!Number.isSafeInteger(inventory[key]) ||
|
|
Number(inventory[key]) < 0)
|
|
)
|
|
throw new Error(`Invalid inventory ${key}.`);
|
|
}
|
|
if (
|
|
inventory.cpuUsagePercent !== undefined &&
|
|
(typeof inventory.cpuUsagePercent !== 'number' ||
|
|
!Number.isFinite(inventory.cpuUsagePercent) ||
|
|
inventory.cpuUsagePercent < 0 ||
|
|
inventory.cpuUsagePercent > 100)
|
|
)
|
|
throw new Error('Invalid inventory cpuUsagePercent.');
|
|
validateInstanceResources(inventory);
|
|
if (inventory.instanceStates !== undefined) {
|
|
if (!Array.isArray(inventory.instanceStates) || inventory.instanceStates.length > 1000) {
|
|
throw new Error('Invalid instance state inventory.');
|
|
}
|
|
// A malformed server observation must not discard the host heartbeat.
|
|
inventory.instanceStates = inventory.instanceStates.filter(
|
|
(state) =>
|
|
state &&
|
|
typeof state === 'object' &&
|
|
typeof state.instanceId === 'string' &&
|
|
/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(state.instanceId) &&
|
|
Number.isSafeInteger(state.generation) &&
|
|
state.generation >= 1 &&
|
|
typeof state.state === 'string' &&
|
|
/^[a-z-]{1,40}$/.test(state.state) &&
|
|
typeof state.healthy === 'boolean' &&
|
|
(!state.healthy || state.state === 'running') &&
|
|
typeof state.observedAt === 'string' &&
|
|
state.observedAt.length <= 40 &&
|
|
Number.isFinite(Date.parse(state.observedAt))
|
|
);
|
|
}
|
|
for (const key of ['dockerRunning', 'directoriesWritable'])
|
|
if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean')
|
|
throw new Error(`Invalid inventory ${key}.`);
|
|
for (const key of ['runtimeImages', 'preparedRuntimeImages'] as const) {
|
|
const images = inventory[key];
|
|
if (
|
|
images !== undefined &&
|
|
(!Array.isArray(images) || images.some((image) => typeof image !== 'string'))
|
|
)
|
|
throw new Error('Invalid runtime image inventory.');
|
|
}
|
|
if (
|
|
inventory.capabilities !== undefined &&
|
|
(!Array.isArray(inventory.capabilities) ||
|
|
inventory.capabilities.length > 32 ||
|
|
inventory.capabilities.some((value) => typeof value !== 'string' || value.length > 80))
|
|
)
|
|
throw new Error('Invalid agent capability inventory.');
|
|
if (inventory.plutoniumCache !== undefined) {
|
|
const cache = inventory.plutoniumCache;
|
|
if (
|
|
!cache ||
|
|
typeof cache !== 'object' ||
|
|
Array.isArray(cache) ||
|
|
!Number.isSafeInteger(cache.revision) ||
|
|
cache.revision < 1 ||
|
|
!Number.isSafeInteger(cache.bytes) ||
|
|
cache.bytes < 0
|
|
)
|
|
throw new Error('Invalid Plutonium cache inventory.');
|
|
inventory.plutoniumCache = { revision: cache.revision, bytes: cache.bytes };
|
|
}
|
|
if (inventory.agentUpdate !== undefined) {
|
|
const update = inventory.agentUpdate;
|
|
const version = (value: unknown) =>
|
|
typeof value === 'string' && value.length <= 80 && /^\d+\.\d+\.\d+$/.test(value);
|
|
if (
|
|
!update ||
|
|
typeof update !== 'object' ||
|
|
Array.isArray(update) ||
|
|
typeof update.enabled !== 'boolean' ||
|
|
!['current', 'checking', 'deferred', 'updating', 'updated', 'failed'].includes(
|
|
update.state
|
|
) ||
|
|
!version(update.currentVersion) ||
|
|
(update.targetVersion !== undefined && !version(update.targetVersion)) ||
|
|
(update.error !== undefined &&
|
|
(typeof update.error !== 'string' || update.error.length > 2000)) ||
|
|
(update.checkedAt !== undefined &&
|
|
(typeof update.checkedAt !== 'string' ||
|
|
update.checkedAt.length > 80 ||
|
|
!Number.isFinite(Date.parse(update.checkedAt))))
|
|
) {
|
|
throw new Error('Invalid agent update inventory.');
|
|
}
|
|
// Only expose display fields, never the local recovery journal or secrets.
|
|
inventory.agentUpdate = {
|
|
enabled: update.enabled,
|
|
state: update.state,
|
|
currentVersion: update.currentVersion,
|
|
...(update.targetVersion === undefined ? {} : { targetVersion: update.targetVersion }),
|
|
...(update.error === undefined ? {} : { error: update.error }),
|
|
...(update.checkedAt === undefined ? {} : { checkedAt: update.checkedAt })
|
|
};
|
|
}
|
|
if (inventory.agentLanes !== undefined) {
|
|
if (!Array.isArray(inventory.agentLanes) || inventory.agentLanes.length > 4)
|
|
throw new Error('Invalid agent lane inventory.');
|
|
// Keep valid lane diagnostics without discarding a host heartbeat over
|
|
// one malformed sample. Do not persist thread details or raw errors.
|
|
const names = new Set<string>();
|
|
inventory.agentLanes = inventory.agentLanes
|
|
.filter((lane) => {
|
|
if (
|
|
!lane ||
|
|
!['health-1', 'health-2', 'read', 'write'].includes(lane.name) ||
|
|
names.has(lane.name) ||
|
|
typeof lane.alive !== 'boolean' ||
|
|
typeof lane.busy !== 'boolean' ||
|
|
typeof lane.pollAgeSeconds !== 'number' ||
|
|
!Number.isFinite(lane.pollAgeSeconds) ||
|
|
lane.pollAgeSeconds < 0 ||
|
|
(lane.lastPollAt !== null &&
|
|
(typeof lane.lastPollAt !== 'string' ||
|
|
lane.lastPollAt.length > 40 ||
|
|
!Number.isFinite(Date.parse(lane.lastPollAt))))
|
|
)
|
|
return false;
|
|
names.add(lane.name);
|
|
return true;
|
|
})
|
|
.map(({ name, alive, busy, lastPollAt, pollAgeSeconds }) => ({
|
|
name,
|
|
alive,
|
|
busy,
|
|
lastPollAt,
|
|
pollAgeSeconds
|
|
}));
|
|
}
|
|
if (
|
|
inventory.libraries !== undefined &&
|
|
(!Array.isArray(inventory.libraries) ||
|
|
inventory.libraries.length > 2000 ||
|
|
inventory.libraries.some(
|
|
(library) =>
|
|
!library ||
|
|
typeof library !== 'object' ||
|
|
typeof library.id !== 'string' ||
|
|
!library.id ||
|
|
library.id.length > 200 ||
|
|
![
|
|
'missing',
|
|
'downloading',
|
|
'verifying',
|
|
'ready',
|
|
'failed',
|
|
'outdated'
|
|
].includes(library.state)
|
|
))
|
|
)
|
|
throw new Error('Invalid game library inventory.');
|
|
return inventory;
|
|
}
|
|
export function enrollHost(
|
|
token: unknown,
|
|
rawInventory: unknown
|
|
): { hostId: string; credential: string; heartbeatSeconds: number } {
|
|
if (typeof token !== 'string' || token.length > 200)
|
|
throw new Error('Enrollment token is invalid or expired.');
|
|
const inventory = validateInventory(rawInventory);
|
|
return database().transaction(() => {
|
|
const enrollment = database()
|
|
.prepare(
|
|
'SELECT id,host_id FROM ec_host_enrollments WHERE token_hash=? AND consumed_at IS NULL AND expires_at>?'
|
|
)
|
|
.get(digest(token), now()) as { id: string; host_id: string } | undefined;
|
|
if (!enrollment) throw new Error('Enrollment token is invalid or expired.');
|
|
const changed = database()
|
|
.prepare(
|
|
'UPDATE ec_host_enrollments SET consumed_at=? WHERE id=? AND consumed_at IS NULL'
|
|
)
|
|
.run(now(), enrollment.id);
|
|
if (changed.changes !== 1) throw new Error('Enrollment token has already been consumed.');
|
|
revokeCredentials(enrollment.host_id);
|
|
const credential = issueCredential(enrollment.host_id);
|
|
database()
|
|
.prepare(
|
|
'UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?'
|
|
)
|
|
.run(
|
|
now(),
|
|
now(),
|
|
inventory.hostname ?? '',
|
|
JSON.stringify(inventory),
|
|
enrollment.host_id
|
|
);
|
|
recordResources(enrollment.host_id, inventory, now());
|
|
scheduleAvailabilityRefresh();
|
|
return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 };
|
|
})();
|
|
}
|
|
export function authenticateCredential(token: string): Host | null {
|
|
if (!token || token.length > 200) return null;
|
|
const row = database()
|
|
.prepare(
|
|
'SELECT id,host_id,last_used_at FROM ec_host_credentials WHERE token_hash=? AND revoked_at IS NULL'
|
|
)
|
|
.get(digest(token)) as
|
|
{ id: string; host_id: string; last_used_at: string | null } | undefined;
|
|
if (!row) return null;
|
|
const timestamp = Date.now();
|
|
if (row.last_used_at === null || timestamp - Date.parse(row.last_used_at) >= 60_000) {
|
|
database()
|
|
.prepare(
|
|
`UPDATE ec_host_credentials SET last_used_at=? WHERE id=? AND revoked_at IS NULL
|
|
AND (last_used_at IS NULL OR last_used_at<=?)`
|
|
)
|
|
.run(
|
|
new Date(timestamp).toISOString(),
|
|
row.id,
|
|
new Date(timestamp - 60_000).toISOString()
|
|
);
|
|
}
|
|
return getHost(row.host_id);
|
|
}
|
|
export function revokeCredentials(hostId: string): void {
|
|
const host = requireHost(hostId);
|
|
if (host.decommission && !host.decommission.retryable)
|
|
throw new Error('Host cleanup needs its credentials until removal is confirmed.');
|
|
database()
|
|
.prepare(
|
|
'UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL'
|
|
)
|
|
.run(now(), hostId);
|
|
scheduleAvailabilityRefresh();
|
|
}
|
|
export function rotateCredential(hostId: string, nextCredential?: string): string {
|
|
requireHost(hostId);
|
|
if (nextCredential !== undefined && !/^ec_host_[A-Za-z0-9_-]{43}$/.test(nextCredential))
|
|
throw new Error('New credentials must contain 32 random bytes encoded as base64url.');
|
|
return database().transaction(() => {
|
|
if (nextCredential) {
|
|
const existing = database()
|
|
.prepare('SELECT host_id,revoked_at FROM ec_host_credentials WHERE token_hash=?')
|
|
.get(digest(nextCredential)) as
|
|
{ host_id: string; revoked_at: string | null } | undefined;
|
|
if (existing) {
|
|
if (existing.host_id !== hostId || existing.revoked_at)
|
|
throw new Error('This credential cannot be reused.');
|
|
// A lost rotation response is retried with the persisted replacement.
|
|
// This acknowledges an active credential without authorizing another rotation.
|
|
return nextCredential;
|
|
}
|
|
}
|
|
const operation = database()
|
|
.prepare(
|
|
`SELECT id FROM ec_host_operations WHERE host_id=? AND kind='credentials.rotate'
|
|
AND state IN ('pending','accepted','running','unknown') ORDER BY created_at,rowid LIMIT 1`
|
|
)
|
|
.get(hostId) as { id: string } | undefined;
|
|
if (!operation)
|
|
throw new Error('Credential rotation requires an open administrator request.');
|
|
revokeCredentials(hostId);
|
|
const credential = issueCredential(hostId, nextCredential);
|
|
reportHostOperationStatus(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { rotated: true }
|
|
});
|
|
return credential;
|
|
})();
|
|
}
|
|
export function heartbeat(hostId: string, value: unknown): Host {
|
|
requireHost(hostId);
|
|
const inventory = validateInventory(value);
|
|
database().transaction(() => {
|
|
const timestamp = now();
|
|
database()
|
|
.prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?')
|
|
.run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId);
|
|
recordResources(hostId, inventory, timestamp);
|
|
})();
|
|
const host = requireHost(hostId);
|
|
refreshHostAvailability(host);
|
|
return host;
|
|
}
|
|
|
|
function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
|
|
recordInstanceResources(hostId, inventory, timestamp);
|
|
recordTemperatures(hostId, inventory, timestamp);
|
|
database()
|
|
.prepare(
|
|
`INSERT OR REPLACE INTO ec_host_resource_samples
|
|
(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,?,?,?,?)`
|
|
)
|
|
.run(
|
|
hostId,
|
|
timestamp,
|
|
inventory.cpuCount ?? null,
|
|
inventory.cpuUsagePercent ?? null,
|
|
inventory.memoryTotal ?? null,
|
|
inventory.memoryAvailable ?? null
|
|
);
|
|
if (fleetSettings().hostResourceKeepDays)
|
|
database()
|
|
.prepare('DELETE FROM ec_host_resource_samples WHERE host_id=? AND sampled_at<?')
|
|
.run(
|
|
hostId,
|
|
new Date(
|
|
Date.parse(timestamp) - fleetSettings().hostResourceKeepDays * 86400_000
|
|
).toISOString()
|
|
);
|
|
}
|
|
export function requireReadyHost(
|
|
hostId: string,
|
|
options: {
|
|
libraryId?: string;
|
|
requiredBytes?: number;
|
|
image?: string;
|
|
port?: number;
|
|
bindAddress?: string;
|
|
protocol?: 'udp' | 'tcp';
|
|
} = {}
|
|
): Host {
|
|
const host = requireHost(hostId);
|
|
const reasons = [...host.readinessReasons];
|
|
if (
|
|
options.requiredBytes !== undefined &&
|
|
(host.inventory.storageAvailable ?? 0) < options.requiredBytes + host.minimumFreeBytes
|
|
)
|
|
reasons.push('Insufficient storage for this deployment.');
|
|
if (
|
|
options.libraryId &&
|
|
!host.inventory.libraries?.some(
|
|
(library) => library.id === options.libraryId && library.state === 'ready'
|
|
)
|
|
)
|
|
reasons.push('The selected game library is not ready on this host.');
|
|
if (options.image && !host.inventory.preparedRuntimeImages?.includes(options.image))
|
|
reasons.push('The selected runtime image is not prepared on this host.');
|
|
if (
|
|
options.port !== undefined &&
|
|
!hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol)
|
|
)
|
|
reasons.push('The selected host port is not available.');
|
|
if (reasons.length) throw new Error(reasons.join(' '));
|
|
return host;
|
|
}
|
|
|
|
export type OperationRow = {
|
|
id: string;
|
|
host_id: string;
|
|
kind: string;
|
|
instance_id: string | null;
|
|
generation: number | null;
|
|
payload: string;
|
|
state: HostOperationState;
|
|
result: string | null;
|
|
error: string | null;
|
|
created_at: string;
|
|
updated_at: string;
|
|
};
|
|
export type OperationDeliveryRow = Omit<OperationRow, 'payload'>;
|
|
export type HostOperationStatus = Omit<HostOperation, 'payload'>;
|
|
const operationStatusColumns =
|
|
'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at';
|
|
const operationPayloadSql =
|
|
'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
|
|
const readOnlyOperationSql = `(o.kind IN ('host.inspect','instance.inspect','rcon.status-batch','files.read','files.list','files.stat','files.manifest','logs.read')
|
|
OR (o.kind='rcon.execute' AND COALESCE(json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version'),0)))`;
|
|
/** Optional cache downloads yield their write lane to deployments and host changes. */
|
|
export function shouldDeferPlutoniumPreparation(hostId: string): boolean {
|
|
return !!database()
|
|
.prepare(
|
|
`SELECT 1 FROM ec_host_operations o WHERE o.host_id=?
|
|
AND o.state IN ('pending','accepted','running','unknown') AND o.kind!='plutonium.prepare'
|
|
AND NOT ${readOnlyOperationSql} LIMIT 1`
|
|
)
|
|
.get(hostId);
|
|
}
|
|
function operationStatusFromRow(row: Omit<OperationRow, 'payload'>): HostOperationStatus {
|
|
return {
|
|
id: row.id,
|
|
hostId: row.host_id,
|
|
kind: row.kind,
|
|
instanceId: row.instance_id,
|
|
generation: row.generation,
|
|
state: row.state,
|
|
result: row.result === null ? null : JSON.parse(row.result),
|
|
error: row.error,
|
|
createdAt: row.created_at,
|
|
updatedAt: row.updated_at
|
|
};
|
|
}
|
|
function operationFromRow(row: OperationRow): HostOperation {
|
|
const status = operationStatusFromRow(row);
|
|
return {
|
|
...status,
|
|
result: unpackFileJson(status.result),
|
|
payload: unpackFileJson(JSON.parse(row.payload))
|
|
};
|
|
}
|
|
export function getHostOperation(id: string): HostOperation | null {
|
|
const row = database()
|
|
.prepare(
|
|
`SELECT ${operationStatusColumns},${operationPayloadSql} AS payload FROM ec_host_operations o WHERE o.id=?`
|
|
)
|
|
.get(id) as OperationRow | undefined;
|
|
return row ? operationFromRow(row) : null;
|
|
}
|
|
/** Polling and receipts need the outcome, never another copy of launch files. */
|
|
export function getHostOperationStatus(id: string): HostOperationStatus | null {
|
|
const row = database()
|
|
.prepare(
|
|
'SELECT id,host_id,kind,instance_id,generation,state,result,error,created_at,updated_at FROM ec_host_operations WHERE id=?'
|
|
)
|
|
.get(id) as Omit<OperationRow, 'payload'> | undefined;
|
|
return row ? operationStatusFromRow(row) : null;
|
|
}
|
|
export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] {
|
|
const columns = summary
|
|
? `o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.error,o.created_at,o.updated_at,'{}' AS payload,NULL AS result`
|
|
: `${operationStatusColumns},${operationPayloadSql} AS payload`;
|
|
return (
|
|
database()
|
|
.prepare(
|
|
`SELECT ${columns} FROM ec_host_operations o WHERE o.host_id=? AND o.history_expired_at IS NULL ORDER BY o.created_at DESC,o.rowid DESC LIMIT ?`
|
|
)
|
|
.all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]
|
|
).map(operationFromRow);
|
|
}
|
|
/** Browse retained history without loading command payloads or credential-bearing results. */
|
|
export function listHostOperationHistory(
|
|
page = 1,
|
|
filter: HostOperationFilter = 'important',
|
|
scope: HostOperationScope = {}
|
|
): HostOperationHistoryPage {
|
|
const handle = database();
|
|
const pageSize = 25;
|
|
const conditions =
|
|
filter === 'all'
|
|
? []
|
|
: [filter === 'routine' ? 'o.routine=1' : `(${importantOperationSql})`];
|
|
conditions.push('o.history_expired_at IS NULL');
|
|
const parameters: string[] = [];
|
|
if (scope.host) {
|
|
conditions.push('o.host_id=?');
|
|
parameters.push(scope.host);
|
|
}
|
|
if (scope.server) {
|
|
conditions.push('o.server_name=?');
|
|
parameters.push(scope.server);
|
|
}
|
|
if (scope.operation) {
|
|
conditions.push('o.id=?');
|
|
parameters.push(scope.operation);
|
|
}
|
|
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
return handle.transaction((): HostOperationHistoryPage => {
|
|
const countColumn = filter === 'all' ? 'total' : filter;
|
|
// An exact operation seeks its unique ID instead of paging through history.
|
|
const total = scope.operation
|
|
? ((
|
|
handle
|
|
.prepare(`SELECT 1 AS total FROM ec_host_operations o ${where}`)
|
|
.get(...parameters) as { total: number } | undefined
|
|
)?.total ?? 0)
|
|
: ((
|
|
handle
|
|
.prepare(
|
|
`SELECT ${countColumn} AS total FROM ec_host_operation_history_counts WHERE host_id=? AND server_name=?`
|
|
)
|
|
.get(scope.host ?? '', scope.server ?? '') as { total: number } | undefined
|
|
)?.total ?? 0);
|
|
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
|
const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1);
|
|
const operations = handle
|
|
.prepare(
|
|
`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind,
|
|
o.instance_id AS instanceId,o.generation,o.state,
|
|
CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0
|
|
THEN COALESCE(NULLIF(o.error,''),'Observation reported a problem') ELSE o.error END AS error,
|
|
o.created_at AS createdAt,o.updated_at AS updatedAt,o.routine,o.server_name AS serverName,o.target_path AS targetPath,o.source
|
|
FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL
|
|
${where}
|
|
ORDER BY o.created_at DESC,o.rowid DESC LIMIT ? OFFSET ?`
|
|
)
|
|
.all(
|
|
...parameters,
|
|
pageSize,
|
|
(currentPage - 1) * pageSize
|
|
) as HostOperationHistoryPage['operations'];
|
|
return {
|
|
operations: operations.map((operation) => ({
|
|
...operation,
|
|
routine: !!operation.routine,
|
|
targetPath: safeOperationTarget(operation.kind, operation.targetPath)
|
|
})),
|
|
filter,
|
|
page: currentPage,
|
|
pageSize,
|
|
total,
|
|
totalPages
|
|
};
|
|
})();
|
|
}
|
|
|
|
/** Bound each cleanup pass so retained history cannot stall monitoring or receipts. */
|
|
export function pruneHostObservations(): number {
|
|
return pruneHistory('observation');
|
|
}
|
|
/** Heartbeats stop either launch generation at its next safe command boundary. */
|
|
export function pendingCreationCancellations(hostId: string): string[] {
|
|
const rows = database()
|
|
.prepare(
|
|
`SELECT json_extract(payload,'$.createOperationId') AS creation_id,
|
|
json_extract(payload,'$.recoveryRebuildOperationId') AS recovery_id FROM ec_host_operations
|
|
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`
|
|
)
|
|
.all(hostId) as { creation_id: string; recovery_id: string | null }[];
|
|
return rows.flatMap((row) =>
|
|
row.recovery_id ? [row.creation_id, row.recovery_id] : [row.creation_id]
|
|
);
|
|
}
|
|
type QueueHostOperationOptions = {
|
|
instanceId?: string;
|
|
generation?: number;
|
|
idempotencyKey?: string;
|
|
id?: string;
|
|
};
|
|
export function queueHostOperation(
|
|
hostId: string,
|
|
kind: string,
|
|
payload: Record<string, unknown>,
|
|
options: QueueHostOperationOptions = {}
|
|
): HostOperation {
|
|
if (!payload || typeof payload !== 'object' || Array.isArray(payload))
|
|
throw new Error('Operation payload must be an object.');
|
|
const operation = enqueueHostOperation(hostId, kind, JSON.stringify(payload), options);
|
|
return getHostOperation(operation.id)!;
|
|
}
|
|
/** Keep immutable launch bytes separate so receipts and delivery do not rewrite large payloads. */
|
|
export function queueHostOperationSerialized(
|
|
hostId: string,
|
|
kind: string,
|
|
payload: Buffer,
|
|
options: QueueHostOperationOptions = {},
|
|
routingPayload: Record<string, unknown> = {}
|
|
): HostOperationStatus {
|
|
if (!Buffer.isBuffer(payload) || payload[0] !== 123 || payload[payload.length - 1] !== 125)
|
|
throw new Error('Serialized operation payload must be a JSON object.');
|
|
if (!routingPayload || typeof routingPayload !== 'object' || Array.isArray(routingPayload))
|
|
throw new Error('Operation routing payload must be an object.');
|
|
return enqueueHostOperation(hostId, kind, payload, options, JSON.stringify(routingPayload));
|
|
}
|
|
function enqueueHostOperation(
|
|
hostId: string,
|
|
kind: string,
|
|
payload: string | Buffer,
|
|
options: QueueHostOperationOptions,
|
|
routingPayload?: string
|
|
): HostOperationStatus {
|
|
const host = requireHost(hostId);
|
|
if (host.decommission)
|
|
throw new Error('Host cleanup is in progress; new operations are blocked.');
|
|
if (kind === 'host.decommission')
|
|
throw new Error('Use the host deletion workflow to request cleanup.');
|
|
if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
|
|
if (
|
|
options.generation !== undefined &&
|
|
(!Number.isSafeInteger(options.generation) || options.generation < 1)
|
|
)
|
|
throw new Error('Instance generation must be a positive integer.');
|
|
const operation = database().transaction(() => {
|
|
if (options.idempotencyKey) {
|
|
const existing = database()
|
|
.prepare(
|
|
`SELECT o.id,o.kind,o.instance_id,o.generation,o.history_expired_at,o.request_hash,${operationPayloadSql}=CAST(? AS TEXT) AS same_payload
|
|
FROM ec_host_operations o WHERE o.host_id=? AND o.idempotency_key=?`
|
|
)
|
|
.get(payload, hostId, options.idempotencyKey) as
|
|
| {
|
|
id: string;
|
|
kind: string;
|
|
instance_id: string | null;
|
|
generation: number | null;
|
|
history_expired_at: string | null;
|
|
request_hash: string | null;
|
|
same_payload: number;
|
|
}
|
|
| undefined;
|
|
if (existing) {
|
|
const samePayload = existing.request_hash
|
|
? existing.request_hash === createHash('sha256').update(payload).digest('hex')
|
|
: !!existing.same_payload;
|
|
if (
|
|
existing.kind !== kind ||
|
|
existing.instance_id !== (options.instanceId ?? null) ||
|
|
existing.generation !== (options.generation ?? null) ||
|
|
!samePayload
|
|
)
|
|
throw new Error(
|
|
'Operation idempotency key is already associated with a different request.'
|
|
);
|
|
return getHostOperationStatus(existing.id)!;
|
|
}
|
|
}
|
|
if (['instance.create', 'create'].includes(kind)) requireReadyHost(hostId);
|
|
if (
|
|
host.mode === 'disabled' &&
|
|
kind !== 'instance.cancel-create' &&
|
|
!(
|
|
host.enablingSince &&
|
|
['runtime.pull', 'library.download', 'plutonium.prepare', 'host.inspect'].includes(
|
|
kind
|
|
)
|
|
)
|
|
)
|
|
throw new Error('Host is disabled. Enable it before requesting operations.');
|
|
const requestHash = createHash('sha256').update(payload).digest('hex');
|
|
const parsedPayload = JSON.parse(payload.toString());
|
|
assertPublicFilePayload(parsedPayload);
|
|
const packed = packFileJson(parsedPayload);
|
|
payload = JSON.stringify(packed.value);
|
|
const id = options.id ?? randomUUID();
|
|
const timestamp = now();
|
|
const serverName = options.instanceId
|
|
? ((
|
|
database()
|
|
.prepare('SELECT server FROM native_instances WHERE id=?')
|
|
.get(options.instanceId) as { server: string } | undefined
|
|
)?.server ?? null)
|
|
: null;
|
|
database()
|
|
.prepare(
|
|
`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name,request_hash)
|
|
VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?,?)`
|
|
)
|
|
.run(
|
|
id,
|
|
hostId,
|
|
kind,
|
|
options.instanceId ?? null,
|
|
options.generation ?? null,
|
|
options.idempotencyKey ?? null,
|
|
routingPayload ?? payload,
|
|
timestamp,
|
|
timestamp,
|
|
(currentOperationSource() ?? 'Controller').slice(0, 200),
|
|
serverName,
|
|
requestHash
|
|
);
|
|
referenceBlobs('operation', id, packed.hashes);
|
|
if (routingPayload !== undefined)
|
|
database()
|
|
.prepare(
|
|
'INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))'
|
|
)
|
|
.run(id, payload);
|
|
database()
|
|
.prepare(
|
|
`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
|
|
target_path=${operationTargetSql} WHERE o.id=?`
|
|
)
|
|
.run(id);
|
|
database()
|
|
.prepare(
|
|
'INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)'
|
|
)
|
|
.run(id, 'pending', timestamp);
|
|
return getHostOperationStatus(id)!;
|
|
})();
|
|
queueMicrotask(() => workEvents.emit(hostId));
|
|
return operation;
|
|
}
|
|
export function pendingHostOperations(
|
|
hostId: string,
|
|
lane: 'all' | 'read' | 'health' | 'write' = 'all'
|
|
): HostOperation[] {
|
|
return pendingHostOperationRows(hostId, lane).map(operationFromRow);
|
|
}
|
|
type OperationLane = 'all' | 'read' | 'health' | 'write';
|
|
export function pendingHostOperationRows(
|
|
hostId: string,
|
|
lane?: OperationLane,
|
|
includePayload?: true
|
|
): OperationRow[];
|
|
export function pendingHostOperationRows(
|
|
hostId: string,
|
|
lane: OperationLane,
|
|
includePayload: false
|
|
): OperationDeliveryRow[];
|
|
/** Delivery snapshots retain status/results while large immutable payloads stay in SQLite. */
|
|
export function pendingHostOperationRows(
|
|
hostId: string,
|
|
lane: OperationLane = 'all',
|
|
includePayload = true
|
|
): OperationDeliveryRow[] {
|
|
const host = requireHost(hostId);
|
|
if ((host.mode === 'disabled' && !host.enablingSince) || !host.connected) return [];
|
|
// Older agents hold activity.lock across the entire delivered batch, including
|
|
// observations. Filtering only the unsupported cleanup can keep the updater
|
|
// busy indefinitely. Drain the old batch by withholding ALL further work;
|
|
// heartbeats and receipts still flow, and the new capability releases this hold.
|
|
// This must live in the controller so the fix also works before an agent update.
|
|
if (
|
|
database()
|
|
.prepare(
|
|
`SELECT 1 FROM ec_host_operations
|
|
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')
|
|
AND (NOT ? OR (json_extract(payload,'$.recoveryRebuildOperationId') IS NOT NULL AND NOT ?)) LIMIT 1`
|
|
)
|
|
.get(
|
|
hostId,
|
|
Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false),
|
|
Number(
|
|
host.inventory.capabilities?.includes('instance.unhealthy-recovery') ?? false
|
|
)
|
|
)
|
|
)
|
|
return [];
|
|
// Accepted/running/unknown commands keep the same immutable ID. The agent
|
|
// consults its durable journal before acknowledging or executing a replay.
|
|
const healthOnly = `(kind IN ('instance.inspect','rcon.status-batch') OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
|
|
const healthLane = host.inventory.capabilities?.includes('operations.health-lane');
|
|
const laneFilter =
|
|
lane === 'all'
|
|
? ''
|
|
: lane === 'health'
|
|
? ` AND ${healthOnly}`
|
|
: ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnlyOperationSql}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;
|
|
// Health never queues behind a delivered file batch. Keep background reads
|
|
// in small batches to amortize delivery without handing out a long backlog.
|
|
const limit = healthLane ? (lane === 'health' ? 1 : lane === 'read' ? 4 : 25) : 25;
|
|
return database().transaction(() => {
|
|
const columns = includePayload
|
|
? `${operationStatusColumns},${operationPayloadSql} AS payload`
|
|
: operationStatusColumns;
|
|
const rows = database()
|
|
.prepare(
|
|
`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
|
|
${laneFilter}
|
|
${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','plutonium.prepare','host.inspect')" : ''}
|
|
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
|
|
AND (kind!='instance.cancel-create' OR ?)
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
|
|
AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cancellation
|
|
JOIN native_provisioning creation ON creation.deployment_id=cancellation.creation_id
|
|
WHERE creation.operation_id=o.id)
|
|
${lane === 'health' ? "AND (state NOT IN ('accepted','running') OR MAX(updated_at,COALESCE(last_delivered_at,'')) <= ?)" : ''}
|
|
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 WHEN kind='plutonium.prepare' THEN 2 ELSE 1 END,
|
|
COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`
|
|
)
|
|
// A short replay lease avoids handing the idle health lane its busy
|
|
// sibling's work, but still recovers a lost receipt or agent restart.
|
|
.all(
|
|
hostId,
|
|
Number(host.readiness === 'ready' || host.mode === 'draining'),
|
|
Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false),
|
|
...(lane === 'health' ? [new Date(Date.now() - 30_000).toISOString()] : [])
|
|
) as OperationDeliveryRow[];
|
|
const preparationLimit = fleetSettings().preparationConcurrency;
|
|
const libraryKinds = ['library.download', 'library.import'];
|
|
const preparing =
|
|
preparationLimit && rows.some((operation) => libraryKinds.includes(operation.kind))
|
|
? new Set(
|
|
(
|
|
database()
|
|
.prepare(
|
|
`SELECT DISTINCT host_id FROM ec_host_operations
|
|
WHERE kind IN ('library.download','library.import') AND last_delivered_at IS NOT NULL
|
|
AND state IN ('pending','accepted','running','unknown')`
|
|
)
|
|
.all() as { host_id: string }[]
|
|
).map((row) => row.host_id)
|
|
)
|
|
: new Set<string>();
|
|
const admitted = rows.filter((operation) => {
|
|
// Withhold unsafe legacy commands without breaking the entire delivery stream.
|
|
if (operation.kind.startsWith('files.') || operation.kind === 'logs.read') {
|
|
try {
|
|
getHostOperationPayload(hostId, operation.id);
|
|
} catch (error) {
|
|
if (error instanceof PrivateFileOperationError) return false;
|
|
throw error;
|
|
}
|
|
}
|
|
if (
|
|
!libraryKinds.includes(operation.kind) ||
|
|
preparing.has(hostId) ||
|
|
!preparationLimit
|
|
)
|
|
return true;
|
|
if (preparing.size >= preparationLimit) return false;
|
|
preparing.add(hostId);
|
|
return true;
|
|
});
|
|
const deliveredAt = now();
|
|
for (const operation of admitted)
|
|
database()
|
|
.prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?')
|
|
.run(deliveredAt, operation.id);
|
|
return admitted;
|
|
})();
|
|
}
|
|
/** Payloads never change after enqueueing; fetch one delivery's UTF-8 bytes without decoding. */
|
|
export function getHostOperationPayload(hostId: string, id: string): Buffer {
|
|
const row = database()
|
|
.prepare(
|
|
`SELECT CAST(${operationPayloadSql} AS BLOB) AS payload FROM ec_host_operations o WHERE o.id=? AND o.host_id=?`
|
|
)
|
|
.get(id, hostId) as { payload: Buffer } | undefined;
|
|
if (!row) throw new Error('Host operation was not found for this host.');
|
|
// Block old private-file commands before delivery to agents predating the guard.
|
|
if (row.payload.includes(BANK_CONFIG_FILE))
|
|
assertPublicFilePayload(JSON.parse(row.payload.toString()));
|
|
if (
|
|
!row.payload.includes('"contentSha256"') ||
|
|
getHost(hostId)?.inventory.capabilities?.includes('files.blobs')
|
|
)
|
|
return row.payload;
|
|
return Buffer.from(JSON.stringify(unpackFileJson(JSON.parse(row.payload.toString()))));
|
|
}
|
|
export function reportHostOperationStatus(
|
|
hostId: string,
|
|
id: string,
|
|
input: { state: unknown; result?: unknown; error?: unknown }
|
|
): HostOperationStatus {
|
|
let liveLogResult: { value: unknown } | undefined;
|
|
const operation = database().transaction(() => {
|
|
requireHost(hostId);
|
|
const operation = getHostOperationStatus(id);
|
|
if (!operation || operation.hostId !== hostId)
|
|
throw new Error('Operation was not found for this host.');
|
|
if (operation.kind === 'host.decommission' && input.state === 'completed')
|
|
throw new Error('Host removal requires the cleanup worker completion receipt.');
|
|
if (
|
|
!['accepted', 'running', 'completed', 'failed', 'unknown'].includes(
|
|
input.state as string
|
|
)
|
|
)
|
|
throw new Error('Invalid operation state.');
|
|
if (['completed', 'failed'].includes(operation.state)) return operation;
|
|
const state = input.state as HostOperationState;
|
|
if (operation.state === 'running' && state === 'accepted') return operation;
|
|
const error =
|
|
input.error === undefined || input.error === null
|
|
? null
|
|
: String(input.error).slice(0, 4000);
|
|
let result = input.result === undefined ? null : JSON.stringify(input.result);
|
|
if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024)
|
|
throw new Error('Operation result exceeds the limit.');
|
|
if (result !== null) {
|
|
const target =
|
|
operation.kind === 'files.read'
|
|
? (database()
|
|
.prepare('SELECT target_path FROM ec_host_operations WHERE id=?')
|
|
.get(id) as { target_path: string | null })
|
|
: null;
|
|
const packed = packFileJson(
|
|
target?.target_path && isPrivateConfigPath(target.target_path)
|
|
? { redacted: true }
|
|
: operation.kind === 'logs.read'
|
|
? logResultMetadata(input.result)
|
|
: input.result
|
|
);
|
|
result = JSON.stringify(packed.value);
|
|
referenceBlobs('operation-result', id, packed.hashes);
|
|
}
|
|
const observation =
|
|
state === 'completed' &&
|
|
!error &&
|
|
successfulObservation(
|
|
operation.kind,
|
|
operation.kind === 'rcon.execute'
|
|
? JSON.parse(getHostOperationPayload(hostId, id).toString('utf8'))
|
|
: {},
|
|
input.result
|
|
);
|
|
database()
|
|
.prepare(
|
|
'UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=?,observation_success=? WHERE id=?'
|
|
)
|
|
.run(state, result, error, now(), Number(observation), id);
|
|
if (operation.kind === 'logs.read' && state === 'completed')
|
|
liveLogResult = { value: input.result ?? null };
|
|
if (operation.kind === 'host.decommission') {
|
|
if (state === 'failed')
|
|
database()
|
|
.prepare(
|
|
"UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?"
|
|
)
|
|
.run(
|
|
error ??
|
|
'The agent could not start cleanup. Repair the installation, then retry deletion.',
|
|
now(),
|
|
id
|
|
);
|
|
else
|
|
database()
|
|
.prepare(
|
|
"UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')"
|
|
)
|
|
.run(now(), id);
|
|
}
|
|
if (operation.state !== state) {
|
|
database()
|
|
.prepare(
|
|
'INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)'
|
|
)
|
|
.run(id, state, now(), error);
|
|
if (
|
|
(operation.kind.startsWith('instance.') && operation.kind !== 'instance.inspect') ||
|
|
operation.kind === 'files.stage'
|
|
) {
|
|
recordAgentReceipt(id, state, now());
|
|
}
|
|
}
|
|
return getHostOperationStatus(id)!;
|
|
})();
|
|
if (liveLogResult)
|
|
for (const reader of logReaders.get(id) ?? []) {
|
|
reader.received = true;
|
|
reader.result = liveLogResult.value;
|
|
}
|
|
queueMicrotask(() => workEvents.emit(`operation:${id}`));
|
|
return operation;
|
|
}
|
|
export function reportHostOperation(
|
|
hostId: string,
|
|
id: string,
|
|
input: { state: unknown; result?: unknown; error?: unknown }
|
|
): HostOperation {
|
|
reportHostOperationStatus(hostId, id, input);
|
|
return getHostOperation(id)!;
|
|
}
|
|
export async function waitHostOperationStatus(
|
|
id: string,
|
|
timeoutMs = 30_000
|
|
): Promise<HostOperationStatus> {
|
|
if (getHostOperationStatus(id)?.kind !== 'logs.read') return waitOperationStatus(id, timeoutMs);
|
|
const reader: { received: boolean; result?: unknown } = { received: false };
|
|
const readers = logReaders.get(id) ?? new Set();
|
|
readers.add(reader);
|
|
logReaders.set(id, readers);
|
|
try {
|
|
const operation = await waitOperationStatus(id, timeoutMs);
|
|
if (operation.state !== 'completed') return operation;
|
|
if (!reader.received)
|
|
throw new Error('Log reply is no longer available. Request a new read.');
|
|
return { ...operation, result: reader.result };
|
|
} finally {
|
|
readers.delete(reader);
|
|
if (!readers.size) logReaders.delete(id);
|
|
}
|
|
}
|
|
async function waitOperationStatus(id: string, timeoutMs: number): Promise<HostOperationStatus> {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (true) {
|
|
const operation = getHostOperationStatus(id);
|
|
if (!operation) throw new Error('Host operation was not found.');
|
|
if (['completed', 'failed', 'unknown'].includes(operation.state)) return operation;
|
|
if (Date.now() >= deadline) {
|
|
// Delivery and expiry both claim the pending row transactionally.
|
|
// Never cancel a delivered check or a command with side effects.
|
|
const expired = database().transaction(() => {
|
|
const timestamp = now();
|
|
const error = 'Expired before delivery.';
|
|
const result = database()
|
|
.prepare(
|
|
`UPDATE ec_host_operations SET state='failed',error=?,updated_at=?
|
|
WHERE id=? AND state='pending' AND last_delivered_at IS NULL
|
|
AND kind IN ('instance.inspect','rcon.status-batch')`
|
|
)
|
|
.run(error, timestamp, id);
|
|
if (result.changes)
|
|
database()
|
|
.prepare(
|
|
'INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)'
|
|
)
|
|
.run(id, 'failed', timestamp, error);
|
|
return result.changes > 0;
|
|
})();
|
|
if (expired) {
|
|
queueMicrotask(() => workEvents.emit(`operation:${id}`));
|
|
throw new Error(`Host operation ${id} expired before delivery.`);
|
|
}
|
|
throw new Error(
|
|
`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`
|
|
);
|
|
}
|
|
await waitForHostWork(
|
|
`operation:${id}`,
|
|
new AbortController().signal,
|
|
Math.min(1000, Math.max(1, deadline - Date.now()))
|
|
);
|
|
}
|
|
}
|
|
export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
|
|
const status = await waitHostOperationStatus(id, timeoutMs);
|
|
const operation = getHostOperation(id)!;
|
|
return status.kind === 'logs.read' ? { ...operation, result: status.result } : operation;
|
|
}
|
|
|
|
export function hostPortAvailable(
|
|
hostId: string,
|
|
candidate: number,
|
|
bindAddress?: string,
|
|
protocol: 'udp' | 'tcp' = 'udp',
|
|
instanceId?: string
|
|
): boolean {
|
|
const host = requireHost(hostId);
|
|
port(candidate);
|
|
const bind = bindAddress ?? host.bindAddress;
|
|
if (host.decommission) return false;
|
|
if (candidate < host.portMin || candidate > host.portMax) return false;
|
|
const rows = database()
|
|
.prepare(
|
|
'SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?'
|
|
)
|
|
.all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[];
|
|
return !rows.some(
|
|
(row) =>
|
|
row.instance_id !== instanceId &&
|
|
(row.bind_address === bind ||
|
|
['0.0.0.0', '::'].includes(bind) ||
|
|
['0.0.0.0', '::'].includes(row.bind_address))
|
|
);
|
|
}
|
|
export function nextHostPort(
|
|
hostId: string,
|
|
additionalPorts: number[] = [],
|
|
bindAddress?: string,
|
|
protocol: 'udp' | 'tcp' = 'udp'
|
|
): number | null {
|
|
const host = requireHost(hostId);
|
|
for (let candidate = host.portMin; candidate <= host.portMax; candidate++)
|
|
if (
|
|
!additionalPorts.includes(candidate) &&
|
|
hostPortAvailable(hostId, candidate, bindAddress, protocol)
|
|
)
|
|
return candidate;
|
|
return null;
|
|
}
|
|
export function reserveHostPort(
|
|
hostId: string,
|
|
instanceId: string,
|
|
options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}
|
|
): number {
|
|
const host = requireHost(hostId);
|
|
const bind = options.bindAddress ?? host.bindAddress;
|
|
const protocol = options.protocol ?? 'udp';
|
|
if (host.decommission)
|
|
throw new Error('Host cleanup is in progress; new port reservations are blocked.');
|
|
if (!isIP(bind)) throw new Error('Bind address must be an IP address.');
|
|
if (!['udp', 'tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.');
|
|
if (!instanceId) throw new Error('An immutable instance ID is required.');
|
|
return database().transaction(() => {
|
|
const existing = database()
|
|
.prepare(
|
|
'SELECT port FROM ec_host_ports WHERE host_id=? AND instance_id=? AND bind_address=? AND protocol=?'
|
|
)
|
|
.get(hostId, instanceId, bind, protocol) as { port: number } | undefined;
|
|
if (existing && (options.port === undefined || options.port === existing.port))
|
|
return existing.port;
|
|
const allocated = options.port ?? nextHostPort(hostId, [], bind, protocol);
|
|
if (allocated === null || !hostPortAvailable(hostId, allocated, bind, protocol, instanceId))
|
|
throw new Error('No available game port on the selected host.');
|
|
database()
|
|
.prepare(
|
|
'INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) VALUES(?,?,?,?,?,?)'
|
|
)
|
|
.run(hostId, instanceId, bind, protocol, allocated, now());
|
|
scheduleAvailabilityRefresh();
|
|
return allocated;
|
|
})();
|
|
}
|
|
export function releaseHostPorts(instanceId: string): void {
|
|
database().prepare('DELETE FROM ec_host_ports WHERE instance_id=?').run(instanceId);
|
|
scheduleAvailabilityRefresh();
|
|
}
|