josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate instances when draining and verify hosts before enabling

Browse Source
This commit is contained in:
josh
2026-10-05 14:21:48 -04:00
parent a142f06c77
commit 16a5bc3534
20 changed files with 525 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/index.ts
+23 -7
View File
@@ -37,6 +37,7 @@ function presentDecommission(row: DecommissionRow | undefined): HostDecommission
error: row.error, retryable: row.stage === 'failed' && getHostOperationStatus(row.operation_id)?.state === 'failed' } : null;
}
function hostFromRow(row: HostRow): Host {
const enabling = database().prepare('SELECT requested_at FROM ec_host_enable_requests WHERE host_id=?').get(row.id) as { requested_at: string } | undefined;
const inventory: HostInventory = JSON.parse(row.inventory);
const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
const reasons: string[] = [];
@@ -53,13 +54,14 @@ function hostFromRow(row: HostRow): Host {
if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.');
if (!libraryReady) reasons.push('No verified game-file library is available yet.');
if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
if (row.mode === 'disabled') reasons.push('Host is disabled.');
if (row.mode === 'disabled') reasons.push(enabling ? 'Host setup must refresh the runtime and verify both BO1 and BO2 game files before enabling.' : 'Host is disabled.');
const decommission = presentDecommission(decommissionRow(row.id));
if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.');
return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
enablingSince: enabling?.requested_at ?? null,
readiness: decommission ? 'deleting' : enabling ? 'preparing' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission };
}
export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
@@ -197,7 +199,7 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
const host = requireHost(id);
if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.');
const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
const mode = input.mode ?? host.mode;
let mode = input.mode ?? host.mode;
if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
const min = port(input.portMin ?? host.portMin); const max = port(input.portMax ?? host.portMax);
if (max < min) throw new Error('The last game port must not precede the first.');
@@ -206,7 +208,20 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
const publicAddress = input.publicAddress === undefined ? host.publicAddress : input.publicAddress === '' ? '' : textField(input.publicAddress, 'Public address');
const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes;
if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) throw new Error('Minimum free storage must be a nonnegative byte count.');
database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
database().transaction(() => {
if (input.mode === 'active' && host.mode === 'draining') throw new Error('Wait for this host to finish draining before enabling it.');
if (input.mode === 'active' && host.mode === 'disabled') {
database().prepare('INSERT OR IGNORE INTO ec_host_enable_requests(host_id,requested_at,operation_floor) VALUES(?,?,(SELECT COALESCE(MAX(rowid),0) FROM ec_host_operations))').run(id, now());
mode = 'disabled';
}
if (input.mode === 'draining') {
if (host.enablingSince) throw new Error('Wait for automatic host setup to finish before draining.');
const incoming = database().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.target_host_id=? AND d.state NOT IN ('done','warning') AND d.server_deleted_at IS NULL LIMIT 1`).get(id);
if (incoming) throw new Error('Wait for incoming instance migrations to finish before draining this host.');
}
database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
})();
scheduleAvailabilityRefresh();
return requireHost(id);
}
@@ -435,7 +450,7 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
}
}
if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.');
const id = options.id ?? randomUUID(); const timestamp = now();
const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?')
.get(options.instanceId) as { server: string } | undefined)?.server ?? null : null;
@@ -461,7 +476,7 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane, in
/** Delivery snapshots retain status/results while large immutable payloads stay in SQLite. */
export function pendingHostOperationRows(hostId: string, lane: OperationLane = 'all', includePayload = true): OperationDeliveryRow[] {
const host = requireHost(hostId);
if (host.mode === 'disabled' || !host.connected) return [];
if ((host.mode === 'disabled' && !host.enablingSince) || !host.connected) return [];
// Older agents hold activity.lock across the entire delivered batch, including
// observations. Filtering only the unsupported cleanup can keep the updater
// busy indefinitely. Drain the old batch by withholding ALL further work;
@@ -484,12 +499,13 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns;
const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
${laneFilter}
${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','host.inspect')" : ''}
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
.all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
.all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
const deliveredAt = now();
for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
return rows;