josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate instances when draining and verify hosts before enabling

Browse Source
This commit is contained in:
josh
2026-10-05 14:21:48 -04:00
parent a142f06c77
commit 16a5bc3534
20 changed files with 525 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/native.ts
+32 -22
View File
@@ -6,7 +6,7 @@ import { posix } from 'node:path';
import { db } from '../db/index.js';
import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
import { getHost, getHostOperationStatus, listHosts, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { getHost, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
@@ -27,6 +27,7 @@ import type { LaunchPreview } from '../../deployment-plan.js';
import type { ProvisioningAction } from '../../provisioning.js';
import { hostAgentUpdatesEnabled } from '../../hosts.js';
import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
import { hostMigration, processMigrationRemoval, reconcileHostDrains } from './host-drain.js';
export type NativeSelection = import('../../hosts.js').NativePlacement;
type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
@@ -258,17 +259,7 @@ function complete(job: Job) {
export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null,
options: { wait?: boolean } = {}): Promise<void> {
// Draining prevents admission immediately and retires existing lobbies only through the normal safety queue.
for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) {
const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
for (const instance of instances) {
if (!getServer(instance.server) || activeBetaRemovalBlocker(instance.server)) continue;
if (db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')").get(instance.server)) continue;
const id = beginDeployment(instance.server, 'destroy', 'host-drain');
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
deploymentStage(id, 'Host draining; waiting for a fresh empty lobby');
}
}
reconcileHostDrains();
const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at,q.selection
FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id
LEFT JOIN native_creation_intents q ON q.deployment_id=d.id
@@ -285,7 +276,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
activeJobs.set(job.id, work);
return work;
});
if (options.wait !== false) await Promise.all(work);
if (options.wait !== false) { await Promise.all(work); reconcileHostDrains(); }
}
const activeJobs = new Map<string, Promise<void>>();
@@ -372,9 +363,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (!instance) return;
job.instance_id = instance.id; job.generation = instance.generation;
}
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
const migration = hostMigration(job.id);
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && !migration && instance.generation === job.generation + 1))) {
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
}
if (migration && !migration.removed_at) {
await processMigrationRemoval(job, instance, migration, () => eligibility(job));
return;
}
const host = getHost(instance.host_id);
if (!host?.connected || host.mode === 'disabled') {
db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
@@ -389,6 +385,17 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
}
if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
if (operation.state === 'failed') {
if (migration) {
if (Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
db().transaction(() => {
db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id);
db().prepare('UPDATE native_provisioning SET operation_id=NULL WHERE deployment_id=?').run(job.id);
db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
})();
}
hold(job, `${operation.error ?? 'Destination creation failed before mutation.'} Retrying after fresh capacity checks.`);
return;
}
// The agent reports failed only when no mutation began. Ambiguous
// post-mutation failures remain unknown and retain their capacity.
db().transaction(() => {
@@ -415,7 +422,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
})();
return;
}
if (job.action === 'rebuild' && instance.generation === job.generation) {
if (job.action === 'rebuild' && !migration && instance.generation === job.generation) {
db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation);
instance = nativeInstance(job.server)!;
}
@@ -503,7 +510,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
}
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
if (job.action === 'create' || migration) choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: false, dispatch: true });
if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
const checkOccupancy = async (): Promise<boolean> => {
@@ -523,7 +530,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
}
return true;
};
if (job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
if (!migration && job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
let prepared: PreparedNativeLaunch | undefined;
@@ -541,8 +548,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
rebuild: job.action === 'rebuild' });
if (job.action === 'rebuild' && !await checkOccupancy()) return;
rebuild: job.action === 'rebuild' && !migration });
if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
}
const commit = db().transaction(() => {
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
@@ -569,7 +576,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
}
assertReadyLibrary(instance!.host_id, instance!.library_id);
if (job.action === 'create') choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
if (job.action === 'create' || migration) choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: false, dispatch: true });
}
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
@@ -590,8 +597,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
}
const operationOptions = { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id };
const kind = `instance.${job.action === 'destroy' ? 'remove' : job.action}`;
const attempt = migration ? (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
WHERE host_id=? AND instance_id=? AND generation=? AND kind='instance.create'`).get(instance!.host_id, instance!.id, instance!.generation) as { n: number }).n : 0;
const operationOptions = { instanceId: instance!.id, generation: instance!.generation,
idempotencyKey: migration ? `${job.id}:migration-create:${attempt}` : job.id };
const kind = `instance.${migration ? 'create' : job.action === 'destroy' ? 'remove' : job.action}`;
const operation = prepared ? queueHostOperationSerialized(instance!.host_id, kind,
Buffer.from(prepared.payloadJson.buffer, prepared.payloadJson.byteOffset, prepared.payloadJson.byteLength), operationOptions,
{ name: instance!.server, game: prepared.recipe.game, libraryId: instance!.library_id })