Migrate instances when draining and verify hosts before enabling
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
|
||||
import { getHost, getHostOperationStatus, listHosts, queueHostOperation, reserveHostPort, updateHost } from '../hosts/index.js';
|
||||
import { choosePlacement } from '../hosts/capacity.js';
|
||||
import { scheduleAvailabilityRefresh } from '../hosts/availability.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
import { getServerKey } from '../fleet/server-keys.js';
|
||||
import { captureRecipePlan, recipeForDeployment, recipeStorage, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
|
||||
import { probePlayerEvidence } from '../fleet/status-evidence.js';
|
||||
import { beginDeployment, deploymentStage } from './history.js';
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import type { NativePlacement } from '../../hosts.js';
|
||||
|
||||
interface Migration {
|
||||
deployment_id: string; source_host_id: string; instance_id: string;
|
||||
target_host_id: string | null; target_port: number | null; target_bind: string | null;
|
||||
remove_operation_id: string | null; removed_at: string | null;
|
||||
}
|
||||
interface Job { id: string; server: string; action: string; state: string; stage: string }
|
||||
type Recipe = LaunchRecipePlan & { native: NativePlacement };
|
||||
|
||||
export function hostMigration(id: string): Migration | undefined {
|
||||
return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as Migration | undefined;
|
||||
}
|
||||
|
||||
/** Drain requests survive controller restarts and never make a source slot available early. */
|
||||
export function reconcileHostDrains(): void {
|
||||
for (const host of listHosts().filter(host => host.mode === 'draining' && !host.decommission)) {
|
||||
db().transaction(() => {
|
||||
// Accepted launches that have no instance yet can be placed elsewhere.
|
||||
const queued = db().prepare(`SELECT q.deployment_id,q.selection FROM native_creation_intents q
|
||||
JOIN server_deployments d ON d.id=q.deployment_id
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
WHERE d.state='queued' AND d.server_deleted_at IS NULL AND d.submitted_at IS NULL
|
||||
AND n.operation_id IS NULL AND n.stage_operation_id IS NULL
|
||||
AND json_extract(q.selection,'$.hostId')=?`).all(host.id) as { deployment_id: string; selection: string }[];
|
||||
for (const row of queued) {
|
||||
const selection: NativePlacement = JSON.parse(row.selection);
|
||||
db().prepare('UPDATE native_creation_intents SET selection=? WHERE deployment_id=?')
|
||||
.run(JSON.stringify({ libraryId: selection.libraryId, image: selection.image, automatic: true, network: {} }), row.deployment_id);
|
||||
}
|
||||
const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
|
||||
for (const instance of instances) {
|
||||
const server = getServer(instance.server);
|
||||
if (!server?.map_slug || activeBetaRemovalBlocker(instance.server)) continue;
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
|
||||
AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(instance.server)) continue;
|
||||
const recipe = { ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), image: instance.image,
|
||||
native: { libraryId: instance.library_id, image: instance.image } };
|
||||
const id = beginDeployment(instance.server, 'rebuild', 'host-drain');
|
||||
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)')
|
||||
.run(id, instance.server, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
||||
const stored = recipeStorage(recipe);
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
|
||||
db().prepare('INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id) VALUES(?,?,?)').run(id, host.id, instance.id);
|
||||
deploymentStage(id, 'Host draining; waiting for an empty lobby and destination capacity');
|
||||
}
|
||||
const unfinished = db().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
|
||||
WHERE m.source_host_id=? AND d.state NOT IN ('done','warning','cancelled') AND d.server_deleted_at IS NULL LIMIT 1`).get(host.id);
|
||||
const ports = db().prepare('SELECT 1 FROM ec_host_ports WHERE host_id=? LIMIT 1').get(host.id);
|
||||
if (!instances.length && !unfinished && !ports) updateHost(host.id, { mode: 'disabled' });
|
||||
})();
|
||||
}
|
||||
}
|
||||
|
||||
/** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */
|
||||
export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration,
|
||||
eligibility: () => string | null): Promise<void> {
|
||||
const hold = (message: string) => deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } });
|
||||
if (migration.remove_operation_id) {
|
||||
const operation = getHostOperationStatus(migration.remove_operation_id);
|
||||
if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
|
||||
// A failed receipt certifies no mutation began. Unknown results keep
|
||||
// their reservation and original operation ID until reconciled.
|
||||
db().transaction(() => {
|
||||
db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, instance.id);
|
||||
db().prepare(`UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL,
|
||||
remove_operation_id=NULL WHERE deployment_id=?`).run(job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
|
||||
scheduleAvailabilityRefresh();
|
||||
})();
|
||||
hold('Source removal failed before mutation; checking destination capacity again before retrying.');
|
||||
return;
|
||||
}
|
||||
if (operation?.state !== 'completed') {
|
||||
hold(operation?.error ?? 'Waiting for confirmed source removal. The original agent operation will reconcile.');
|
||||
return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
if (hostMigration(job.id)?.removed_at) return;
|
||||
const current = nativeInstance(job.server);
|
||||
if (current?.id !== instance.id || current.generation !== instance.generation || current.host_id !== migration.source_host_id) throw new Error('Migration source changed.');
|
||||
const target = getHost(migration.target_host_id!);
|
||||
if (!target) throw new Error('Migration destination is unavailable.');
|
||||
const recipe = recipeForDeployment(job.id)!.recipe as Recipe;
|
||||
recipe.native = { libraryId: instance.library_id, image: instance.image, hostId: target.id,
|
||||
bindAddress: migration.target_bind!, publicAddress: target.publicAddress };
|
||||
const stored = recipeStorage(recipe);
|
||||
db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
|
||||
db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(instance.host_id, instance.id);
|
||||
db().prepare(`UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?`)
|
||||
.run(target.id, migration.target_bind, migration.target_port, target.publicAddress, instance.id);
|
||||
db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
|
||||
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
|
||||
db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
|
||||
last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,active_since=NULL,inactive_since=NULL,
|
||||
game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
|
||||
db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
|
||||
deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { phase: 'startup-queue', blocker: null });
|
||||
scheduleAvailabilityRefresh();
|
||||
})();
|
||||
return;
|
||||
}
|
||||
if (!getHost(instance.host_id)?.connected) { hold('Waiting for the source host to reconnect.'); return; }
|
||||
const reason = eligibility() ?? activeBetaRemovalBlocker(job.server);
|
||||
if (reason) { hold(reason); return; }
|
||||
if (!getServerKey(job.server)) { hold('Save a server key before migrating this instance.'); return; }
|
||||
const options = { libraryId: instance.library_id, image: instance.image, thorough: true, dispatch: true };
|
||||
// No source mutation is submitted unless another host can take this instance.
|
||||
choosePlacement(options);
|
||||
const health = await nativeRequest<{ state: string }>(instance, 'instance.inspect');
|
||||
if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
|
||||
const occupancy = await probePlayerEvidence(job.server);
|
||||
if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; }
|
||||
}
|
||||
db().transaction(() => {
|
||||
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
|
||||
if (!state || !['waiting','queued','interrupted'].includes(state.state) || hostMigration(job.id)?.remove_operation_id) return;
|
||||
const current = nativeInstance(job.server);
|
||||
if (current?.id !== instance.id || current.generation !== instance.generation || getHost(instance.host_id)?.mode !== 'draining') return;
|
||||
const blocker = eligibility() ?? activeBetaRemovalBlocker(job.server);
|
||||
if (blocker) { hold(blocker); return; }
|
||||
const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
|
||||
if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
|
||||
const { host, port } = choosePlacement(options);
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
|
||||
const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
|
||||
WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?`)
|
||||
.get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number }).n;
|
||||
const operation = queueHostOperation(instance.host_id, 'instance.remove', {}, {
|
||||
instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:migrate-remove:${attempt}`
|
||||
});
|
||||
db().prepare('UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?')
|
||||
.run(host.id, port, host.bindAddress, operation.id, job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
|
||||
deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { phase: 'container-action', blocker: null });
|
||||
})();
|
||||
}
|
||||
Reference in New Issue
Block a user