Add safe manual instance rebalancing
This commit is contained in:
@@ -2,6 +2,15 @@ import { pauseScalingForOperation } from '../autoscaling/ownership.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureHostSchema } from '../hosts/schema.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { requestRebalanceStopForOperation } from '../rebalancing/admission.js';
|
||||
|
||||
export const rebalanceCancellationGuard = `NOT (d.triggered_by LIKE 'rebalance:%' AND EXISTS (
|
||||
SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id AND m.remove_operation_id IS NOT NULL))`;
|
||||
|
||||
export function rebalanceCancellationBlocker(id: string): string | null {
|
||||
return db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id=? AND NOT (${rebalanceCancellationGuard})`).get(id)
|
||||
? 'This rebalance move has committed source removal and must finish recreating the lobby. Stop the rebalance run to withdraw its remaining moves.' : null;
|
||||
}
|
||||
|
||||
export function provisioningCancelled(id: string): boolean {
|
||||
return !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND state='cancelled'").get(id);
|
||||
@@ -12,17 +21,20 @@ export function assertProvisioningNotCancelled(id: string): void {
|
||||
}
|
||||
|
||||
export function canCancelProvisioningJob(id: string): boolean {
|
||||
return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
|
||||
AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(id);
|
||||
return !!db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id=? AND d.server_deleted_at IS NULL
|
||||
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND ${rebalanceCancellationGuard}`).get(id);
|
||||
}
|
||||
|
||||
/** Cancel remaining work, not effects already performed by the host. */
|
||||
export function cancelProvisioningJob(id: string, reason = 'Cancelled by operator'): boolean {
|
||||
export function cancelProvisioningJob(id: string, reason = 'Cancelled by operator', options: { preserveRebalanceRun?: boolean } = {}): boolean {
|
||||
ensureHostSchema(db());
|
||||
return db().transaction(() => {
|
||||
if (provisioningCancelled(id)) return true;
|
||||
if (!canCancelProvisioningJob(id)) return false;
|
||||
pauseScalingForOperation(id);
|
||||
const rebalance = !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND triggered_by LIKE 'rebalance:%'").get(id);
|
||||
if (rebalance) {
|
||||
if (!options.preserveRebalanceRun) requestRebalanceStopForOperation(id);
|
||||
} else pauseScalingForOperation(id);
|
||||
const at = new Date().toISOString();
|
||||
// Only commands never delivered can be withdrawn. Delivered commands retain
|
||||
// their immutable IDs and receipts; replacements wait for their resolution.
|
||||
|
||||
@@ -4,6 +4,9 @@ import { fleetSettings } from '../settings.js';
|
||||
|
||||
// A preparation owns capacity across asynchronous launch-file generation.
|
||||
export const preparingJobs = new Set<string>();
|
||||
/** A committed rebalance keeps its startup reservation while between source removal and creation. */
|
||||
export const committedRebalanceStartup = `(triggered_by LIKE 'rebalance:%' AND state IN ('waiting','interrupted')
|
||||
AND EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=server_deployments.id AND m.remove_operation_id IS NOT NULL))`;
|
||||
interface Holder { id: string; action: string; host_id: string | null; target_host_id: string | null; removing: number }
|
||||
|
||||
export function lifecycleHolders(): Holder[] {
|
||||
|
||||
@@ -11,7 +11,8 @@ import { readProgress, recordProgress, type ProgressUpdate } from './progress.js
|
||||
import { bindNativeOperation } from './native.js';
|
||||
import { workerActivity } from '../pipeline/activity.js';
|
||||
import { canCancelCreation } from './cancel-creation.js';
|
||||
import { canCancelProvisioningJob, provisioningCancelled } from './cancellation.js';
|
||||
import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js';
|
||||
import { committedRebalanceStartup } from './concurrency.js';
|
||||
import { nativeDeployment } from '../backends/native.js';
|
||||
import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
|
||||
import { serverTitle } from '../../ui/server-labels.js';
|
||||
@@ -23,7 +24,7 @@ export function beginDeployment(server: string, action: ProvisioningAction, trig
|
||||
db().transaction(() => {
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by, requested_by)
|
||||
VALUES (?, ?, ?, 'running', 'Starting', ?, ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy,
|
||||
triggeredBy === 'ui' ? currentRequestUsername() : null);
|
||||
triggeredBy === 'ui' || triggeredBy.startsWith('rebalance:') ? currentRequestUsername() : null);
|
||||
bindNativeOperation(id, server);
|
||||
recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested',
|
||||
label: action === 'create' ? 'Server creation requested'
|
||||
@@ -63,6 +64,11 @@ function warningMessages(raw: string): string[] {
|
||||
|
||||
function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
|
||||
const id = source.source_id;
|
||||
if (source.kind === 'rebalance') {
|
||||
const exists = !!db().prepare('SELECT 1 FROM rebalancing_runs WHERE id=?').get(id);
|
||||
return { ...source, label: exists ? 'Rebalance run' : 'Rebalance history expired',
|
||||
href: exists ? `/capacity?rebalance=1&rebalanceRun=${encodeURIComponent(id)}` : null };
|
||||
}
|
||||
if (source.kind === 'autoscaling') {
|
||||
const exists = !!db().prepare('SELECT 1 FROM autoscaling_policies WHERE id=?').get(id);
|
||||
return { ...source, label: exists ? 'Autoscaling policy' : 'Deleted autoscaling policy', href: exists ? `/autoscaling#${encodeURIComponent(id)}` : null };
|
||||
@@ -94,7 +100,7 @@ function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): P
|
||||
const pools: ProvisioningCapacity['pools'] = [];
|
||||
if (deployment.action === 'create' || deployment.action === 'rebuild') {
|
||||
const jobs = db().prepare(`SELECT id, server, state FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health')
|
||||
WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup})
|
||||
ORDER BY started_at, rowid`).all() as { id: string; server: string; state: string }[];
|
||||
const holders = jobs.filter(job => job.state !== 'queued').map(({ id, server }) => ({ id, server }));
|
||||
const order = jobs.filter(job => job.state === 'queued').findIndex(job => job.id === deployment.id);
|
||||
@@ -127,6 +133,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
|
||||
const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
|
||||
const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
|
||||
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
|
||||
if (row.triggered_by.startsWith('rebalance:')) sources.push(sourceDetails({ kind: 'rebalance', source_id: row.triggered_by.slice('rebalance:'.length) }));
|
||||
const launch = recipeSummaryForDeployment(id);
|
||||
const collection = db().prepare(`SELECT c.id, c.name FROM (
|
||||
SELECT collection_id, deployment_id FROM server_collection_members
|
||||
@@ -271,7 +278,8 @@ const deploymentListColumns = `d.id, d.server, d.server_deleted_at, d.action, d.
|
||||
CASE WHEN json_valid(d.observation_json) THEN json_extract(d.observation_json,'$.blocker.code') END AS waitCode,
|
||||
(d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists,
|
||||
(d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval,
|
||||
(d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')) AS canCancel
|
||||
(d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
|
||||
AND ${rebalanceCancellationGuard}) AS canCancel
|
||||
`;
|
||||
|
||||
function deploymentListQuery(options: ProvisioningListOptions) {
|
||||
@@ -286,8 +294,9 @@ function deploymentListQuery(options: ProvisioningListOptions) {
|
||||
OR ${kinds.map(kind => `d.triggered_by LIKE '${kind}:%'`).join(' OR ')})`;
|
||||
if (options.source === 'campaign' || options.source === 'beta') conditions.push(linked([options.source]));
|
||||
else if (options.source === 'maintenance') conditions.push(linked(['maintenance', 'watchdog']));
|
||||
else if (options.source === 'rebalance') conditions.push("d.triggered_by LIKE 'rebalance:%'");
|
||||
else if (options.source === 'manual' || options.source === 'system') {
|
||||
conditions.push(`NOT ${linked(['campaign', 'beta', 'maintenance', 'watchdog'])}`);
|
||||
conditions.push(`NOT ${linked(['campaign', 'beta', 'maintenance', 'watchdog', 'rebalance'])}`);
|
||||
conditions.push(options.source === 'manual' ? "d.triggered_by='ui'" : "d.triggered_by!='ui'");
|
||||
}
|
||||
const search = (options.server ?? '').slice(0, 100).toLowerCase();
|
||||
@@ -332,7 +341,7 @@ function deploymentListRows(query: ReturnType<typeof deploymentListQuery>, activ
|
||||
const cache = new Map<string, ProvisioningSource>();
|
||||
for (const row of rows) {
|
||||
const sources = byOperation.get(row.id) ?? [];
|
||||
const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling):(.+)$/);
|
||||
const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling|rebalance):(.+)$/);
|
||||
if (fallback && !sources.some(source => source.kind === fallback[1] && source.source_id === fallback[2])) sources.unshift({ kind: fallback[1] as ProvisioningSource['kind'], source_id: fallback[2] });
|
||||
row.sources = sources.map(source => {
|
||||
const key = `${source.kind}:${source.source_id}`;
|
||||
|
||||
@@ -1,29 +1,13 @@
|
||||
import { concurrencyBlocker } from './concurrency.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { provisioningCancelled } from './cancellation.js';
|
||||
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
|
||||
import { getHost, getHostOperationStatus, listHosts, queueHostOperation, reserveHostPort, updateHost } from '../hosts/index.js';
|
||||
import { choosePlacement } from '../hosts/capacity.js';
|
||||
import { scheduleAvailabilityRefresh } from '../hosts/availability.js';
|
||||
import type { NativeInstance } from '../backends/native.js';
|
||||
import { listHosts, updateHost } from '../hosts/index.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
import { getServerKey } from '../fleet/server-keys.js';
|
||||
import { captureRecipePlan, recipeForDeployment, recipeStorage, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
|
||||
import { probePlayerEvidence } from '../fleet/status-evidence.js';
|
||||
import { captureRecipePlan, recipeStorage } from '../fleet/launch-recipes.js';
|
||||
import { beginDeployment, deploymentStage } from './history.js';
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import type { NativePlacement } from '../../hosts.js';
|
||||
|
||||
interface Migration {
|
||||
deployment_id: string; source_host_id: string; instance_id: string;
|
||||
target_host_id: string | null; target_port: number | null; target_bind: string | null;
|
||||
remove_operation_id: string | null; removed_at: string | null;
|
||||
}
|
||||
interface Job { id: string; server: string; action: string; state: string; stage: string }
|
||||
type Recipe = LaunchRecipePlan & { native: NativePlacement };
|
||||
|
||||
export function hostMigration(id: string): Migration | undefined {
|
||||
return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as Migration | undefined;
|
||||
}
|
||||
export { captureMoveRecipe } from './move-recipes.js';
|
||||
export { hostMigration, isRebalanceMigration, processMigrationRemoval, requestInstanceMove } from './migrations.js';
|
||||
|
||||
/** Drain requests survive controller restarts and never make a source slot available early. */
|
||||
export function reconcileHostDrains(): void {
|
||||
@@ -47,7 +31,8 @@ export function reconcileHostDrains(): void {
|
||||
if (!server?.map_slug || activeBetaRemovalBlocker(instance.server)) continue;
|
||||
// An explicit cancellation withdraws this instance's current drain attempt.
|
||||
const latest = db().prepare(`SELECT d.state FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
|
||||
WHERE m.instance_id=? AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(instance.id) as { state: string } | undefined;
|
||||
WHERE m.instance_id=? AND m.source_host_id=? AND d.triggered_by='host-drain'
|
||||
AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(instance.id, host.id) as { state: string } | undefined;
|
||||
if (latest?.state === 'cancelled') continue;
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
|
||||
AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(instance.server)) continue;
|
||||
@@ -69,94 +54,3 @@ export function reconcileHostDrains(): void {
|
||||
})();
|
||||
}
|
||||
}
|
||||
|
||||
/** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */
|
||||
export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration,
|
||||
eligibility: () => string | null): Promise<void> {
|
||||
if (provisioningCancelled(job.id)) return;
|
||||
const hold = (message: string) => { if (!provisioningCancelled(job.id)) deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); };
|
||||
if (migration.remove_operation_id) {
|
||||
const operation = getHostOperationStatus(migration.remove_operation_id);
|
||||
if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
|
||||
// A failed receipt certifies no mutation began. Unknown results keep
|
||||
// their reservation and original operation ID until reconciled.
|
||||
db().transaction(() => {
|
||||
db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, instance.id);
|
||||
db().prepare(`UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL,
|
||||
remove_operation_id=NULL WHERE deployment_id=?`).run(job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
|
||||
scheduleAvailabilityRefresh();
|
||||
})();
|
||||
hold('Source removal failed before mutation; checking destination capacity again before retrying.');
|
||||
return;
|
||||
}
|
||||
if (operation?.state !== 'completed') {
|
||||
hold(operation?.error ?? 'Waiting for confirmed source removal. The original agent operation will reconcile.');
|
||||
return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
if (hostMigration(job.id)?.removed_at) return;
|
||||
const current = nativeInstance(job.server);
|
||||
if (current?.id !== instance.id || current.generation !== instance.generation || current.host_id !== migration.source_host_id) throw new Error('Migration source changed.');
|
||||
const target = getHost(migration.target_host_id!);
|
||||
if (!target) throw new Error('Migration destination is unavailable.');
|
||||
const recipe = recipeForDeployment(job.id)!.recipe as Recipe;
|
||||
recipe.native = { libraryId: instance.library_id, image: instance.image, hostId: target.id,
|
||||
bindAddress: migration.target_bind!, publicAddress: target.publicAddress };
|
||||
const stored = recipeStorage(recipe);
|
||||
db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
|
||||
db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(instance.host_id, instance.id);
|
||||
db().prepare(`UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?`)
|
||||
.run(target.id, migration.target_bind, migration.target_port, target.publicAddress, instance.id);
|
||||
db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
|
||||
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
|
||||
db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
|
||||
last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL,
|
||||
game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
|
||||
db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
|
||||
deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { phase: 'startup-queue', blocker: null });
|
||||
scheduleAvailabilityRefresh();
|
||||
})();
|
||||
return;
|
||||
}
|
||||
if (!getHost(instance.host_id)?.connected) { hold('Waiting for the source host to reconnect.'); return; }
|
||||
const reason = eligibility() ?? activeBetaRemovalBlocker(job.server);
|
||||
if (reason) { hold(reason); return; }
|
||||
if (!getServerKey(job.server)) { hold('Save a server key before migrating this instance.'); return; }
|
||||
const options = { libraryId: instance.library_id, image: instance.image, thorough: true, dispatch: true };
|
||||
// No source mutation is submitted unless another host can take this instance.
|
||||
choosePlacement(options);
|
||||
const health = await nativeRequest<{ state: string }>(instance, 'instance.inspect');
|
||||
if (provisioningCancelled(job.id)) return;
|
||||
if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
|
||||
const occupancy = await probePlayerEvidence(job.server);
|
||||
if (provisioningCancelled(job.id)) return;
|
||||
if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; }
|
||||
}
|
||||
db().transaction(() => {
|
||||
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
|
||||
if (!state || !['waiting','queued','interrupted'].includes(state.state) || hostMigration(job.id)?.remove_operation_id) return;
|
||||
const current = nativeInstance(job.server);
|
||||
if (current?.id !== instance.id || current.generation !== instance.generation || getHost(instance.host_id)?.mode !== 'draining') return;
|
||||
const blocker = eligibility() ?? activeBetaRemovalBlocker(job.server);
|
||||
if (blocker) { hold(blocker); return; }
|
||||
const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
|
||||
if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
|
||||
const { host, port } = choosePlacement(options);
|
||||
const capacityBlocker = concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? concurrencyBlocker(job.id, 'rebuild', host.id);
|
||||
if (capacityBlocker) { hold(capacityBlocker); return; }
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
|
||||
const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
|
||||
WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?`)
|
||||
.get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number }).n;
|
||||
const operation = queueHostOperation(instance.host_id, 'instance.remove', {}, {
|
||||
instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:migrate-remove:${attempt}`
|
||||
});
|
||||
db().prepare('UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?')
|
||||
.run(host.id, port, host.bindAddress, operation.id, job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
|
||||
deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { phase: 'container-action', blocker: null });
|
||||
})();
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@ export interface NativeLaunchInput {
|
||||
rconPassword: string;
|
||||
rebuild: boolean;
|
||||
safetyOverride?: boolean;
|
||||
/** Rebalance replays recorded deployed bytes instead of current promoted files. */
|
||||
frozenRecipeId?: string;
|
||||
}
|
||||
|
||||
export interface PreparedNativeLaunch {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { parentPort } from 'node:worker_threads';
|
||||
import { posix } from 'node:path';
|
||||
import { db } from '../db/index.js';
|
||||
import { relDestForKind } from '../config.js';
|
||||
import { captureRecipe, summarizeRecipe } from '../fleet/launch-recipes.js';
|
||||
import { captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
|
||||
import { sha256Buffer } from '../fleet/files.js';
|
||||
import type { NativeLaunchInput } from './launch-preparation.js';
|
||||
import { choosePlacement, type PlacementOptions } from '../hosts/capacity.js';
|
||||
@@ -17,12 +17,26 @@ parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeL
|
||||
}
|
||||
const input = message.input;
|
||||
// All file versions and composed configuration belong to one snapshot.
|
||||
const recipe = db().transaction(() => ({ ...captureRecipe(input.game, input.mapSlug, input.rconPassword),
|
||||
native: input.native, image: input.instance.image }))();
|
||||
const { recipe, password } = db().transaction(() => {
|
||||
if (!input.frozenRecipeId) return { recipe: { ...captureRecipe(input.game, input.mapSlug, input.rconPassword),
|
||||
native: input.native, image: input.instance.image }, password: input.rconPassword };
|
||||
const stored = recipeForDeployment(input.frozenRecipeId)?.recipe;
|
||||
const migration = db().prepare('SELECT runtime_password FROM ec_host_migrations WHERE deployment_id=?')
|
||||
.get(input.frozenRecipeId) as { runtime_password: string | null } | undefined;
|
||||
if (!stored || !migration?.runtime_password || stored.files.some(file => file.content === undefined)) {
|
||||
throw new Error('The preserved deployed recipe or runtime credentials are unavailable.');
|
||||
}
|
||||
for (const file of stored.files) {
|
||||
const bytes = Buffer.from(file.content!, 'base64');
|
||||
if (bytes.length !== file.size || sha256Buffer(bytes) !== file.sha256) throw new Error('Preserved deployed file verification failed.');
|
||||
}
|
||||
return { recipe: { ...stored, native: input.native, image: input.instance.image } as LaunchRecipe,
|
||||
password: migration.runtime_password };
|
||||
})();
|
||||
const instance = input.instance;
|
||||
const payload = {
|
||||
name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image,
|
||||
port: instance.port, bindAddress: instance.bind_address, rconPassword: input.rconPassword,
|
||||
port: instance.port, bindAddress: instance.bind_address, rconPassword: password,
|
||||
env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port),
|
||||
PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', PLUTAINER_CONFIG_FILE: recipe.configFile,
|
||||
PLUTAINER_MAP_ROTATE: 'false',
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import { concurrencyBlocker, preparingJobs, committedRebalanceStartup } from './concurrency.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { provisioningCancelled } from './cancellation.js';
|
||||
import { nativeInstance, nativeRequest, runtimePasswordForServer, type NativeInstance } from '../backends/native.js';
|
||||
import { getHost, getHostOperationStatus, queueHostOperation, reserveHostPort } from '../hosts/index.js';
|
||||
import { choosePlacement } from '../hosts/capacity.js';
|
||||
import { scheduleAvailabilityRefresh } from '../hosts/availability.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
import { getServerKey } from '../fleet/server-keys.js';
|
||||
import { recipeForDeployment, recipeStorage, recipeSummaryForDeployment, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
|
||||
import { captureMoveRecipe, moveRecipeMatches, moveRecipeStorage } from './move-recipes.js';
|
||||
import { rebalanceAdmission } from '../rebalancing/admission.js';
|
||||
import { probePlayerEvidence } from '../fleet/status-evidence.js';
|
||||
import { beginDeployment, deploymentStage } from './history.js';
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import type { NativePlacement } from '../../hosts.js';
|
||||
|
||||
interface Migration {
|
||||
deployment_id: string; source_host_id: string; instance_id: string;
|
||||
target_host_id: string | null; target_port: number | null; target_bind: string | null;
|
||||
remove_operation_id: string | null; removed_at: string | null;
|
||||
}
|
||||
interface Job { id: string; server: string; action: string; state: string; stage: string }
|
||||
type Recipe = LaunchRecipePlan & { native: NativePlacement };
|
||||
export { captureMoveRecipe } from './move-recipes.js';
|
||||
|
||||
export function isRebalanceMigration(id: string): boolean {
|
||||
return !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND triggered_by LIKE 'rebalance:%'").get(id);
|
||||
}
|
||||
|
||||
/** Queue a move to one explicit host; its source remains active throughout. */
|
||||
export function requestInstanceMove(serverName: string, targetHostId: string,
|
||||
options: { triggeredBy: string; recipe?: LaunchRecipePlan }): string {
|
||||
return db().transaction(() => {
|
||||
const instance = nativeInstance(serverName);
|
||||
const server = getServer(serverName);
|
||||
if (!instance || !server?.map_slug) throw new Error('The native instance is unavailable.');
|
||||
if (instance.host_id === targetHostId) throw new Error('Choose a different destination host.');
|
||||
if (getHost(instance.host_id)?.mode !== 'active') throw new Error('The source host must be active.');
|
||||
if (!options.triggeredBy.startsWith('rebalance:')) throw new Error('An instance move requires a rebalance run.');
|
||||
const blocker = activeBetaRemovalBlocker(serverName);
|
||||
if (blocker) throw new Error(blocker);
|
||||
if (!getServerKey(serverName)) throw new Error('Save a server key before moving this instance.');
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
|
||||
AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(serverName)) {
|
||||
throw new Error('Another provisioning operation is active for this instance.');
|
||||
}
|
||||
const recipe = options.recipe ?? captureMoveRecipe(serverName);
|
||||
if (recipe.game !== server.game || recipe.mapSlug !== server.map_slug || recipe.image !== instance.image
|
||||
|| !moveRecipeMatches(serverName, recipe)) throw new Error('The deployed files changed; generate a fresh rebalance preview.');
|
||||
const password = runtimePasswordForServer(serverName);
|
||||
if (!password) throw new Error('The deployed runtime credentials are unavailable.');
|
||||
choosePlacement({ hostId: targetHostId, excludeHostIds: [instance.host_id], libraryId: instance.library_id,
|
||||
image: instance.image, thorough: true, dispatch: true });
|
||||
const id = beginDeployment(serverName, 'rebuild', options.triggeredBy);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)')
|
||||
.run(id, serverName, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
||||
const stored = moveRecipeStorage({ ...recipe, native: { libraryId: instance.library_id, image: instance.image, hostId: targetHostId } });
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
|
||||
db().prepare('INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,runtime_password) VALUES(?,?,?,?)')
|
||||
.run(id, instance.host_id, instance.id, password);
|
||||
deploymentStage(id, 'Rebalance queued; waiting to confirm an empty lobby and destination capacity', { phase: 'eligibility', recipeStatus: 'queued' });
|
||||
return id;
|
||||
})();
|
||||
}
|
||||
|
||||
export function hostMigration(id: string): Migration | undefined {
|
||||
return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as Migration | undefined;
|
||||
}
|
||||
|
||||
/** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */
|
||||
export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration,
|
||||
eligibility: () => string | null): Promise<void> {
|
||||
if (provisioningCancelled(job.id)) return;
|
||||
const hold = (message: string) => { if (!provisioningCancelled(job.id)) deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); };
|
||||
if (migration.remove_operation_id) {
|
||||
const operation = getHostOperationStatus(migration.remove_operation_id);
|
||||
if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
|
||||
// A failed receipt certifies no mutation began. Unknown results keep
|
||||
// their reservation and original operation ID until reconciled.
|
||||
db().transaction(() => {
|
||||
db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, instance.id);
|
||||
db().prepare(`UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL,
|
||||
remove_operation_id=NULL WHERE deployment_id=?`).run(job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
|
||||
scheduleAvailabilityRefresh();
|
||||
})();
|
||||
hold('Source removal failed before mutation; checking destination capacity again before retrying.');
|
||||
return;
|
||||
}
|
||||
if (operation?.state !== 'completed') {
|
||||
hold(operation?.error ?? 'Waiting for confirmed source removal. The original agent operation will reconcile.');
|
||||
return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
if (hostMigration(job.id)?.removed_at) return;
|
||||
const current = nativeInstance(job.server);
|
||||
if (current?.id !== instance.id || current.generation !== instance.generation || current.host_id !== migration.source_host_id) throw new Error('Migration source changed.');
|
||||
const target = getHost(migration.target_host_id!);
|
||||
if (!target) throw new Error('Migration destination is unavailable.');
|
||||
const rebalance = isRebalanceMigration(job.id);
|
||||
const recipe = (rebalance ? { format: 1, ...recipeSummaryForDeployment(job.id)!.recipe }
|
||||
: recipeForDeployment(job.id)!.recipe) as Recipe;
|
||||
recipe.native = { libraryId: instance.library_id, image: instance.image, hostId: target.id,
|
||||
bindAddress: migration.target_bind!, publicAddress: target.publicAddress };
|
||||
const stored = rebalance ? moveRecipeStorage(recipe) : recipeStorage(recipe);
|
||||
db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
|
||||
db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(instance.host_id, instance.id);
|
||||
db().prepare(`UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?`)
|
||||
.run(target.id, migration.target_bind, migration.target_port, target.publicAddress, instance.id);
|
||||
db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
|
||||
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
|
||||
db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
|
||||
last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL,
|
||||
game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
|
||||
db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
|
||||
deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { phase: 'startup-queue', blocker: null });
|
||||
scheduleAvailabilityRefresh();
|
||||
})();
|
||||
return;
|
||||
}
|
||||
if (!getHost(instance.host_id)?.connected) { hold('Waiting for the source host to reconnect.'); return; }
|
||||
const reason = rebalanceAdmission(job.id) ?? eligibility() ?? activeBetaRemovalBlocker(job.server);
|
||||
if (reason) { hold(reason); return; }
|
||||
if (!getServerKey(job.server)) { hold('Save a server key before migrating this instance.'); return; }
|
||||
const rebalance = isRebalanceMigration(job.id);
|
||||
const recipe = rebalance ? recipeSummaryForDeployment(job.id)!.recipe : null;
|
||||
if (rebalance && !moveRecipeMatches(job.server, { format: 1, ...recipe! })) {
|
||||
hold('The deployed files changed; stop this run and generate a fresh rebalance preview.'); return;
|
||||
}
|
||||
const options = { libraryId: instance.library_id, image: instance.image, thorough: true, dispatch: true,
|
||||
excludeHostIds: [instance.host_id], ...(rebalance ? { hostId: recipe!.native!.hostId } : {}) };
|
||||
// No source mutation is submitted unless another host can take this instance.
|
||||
choosePlacement(options);
|
||||
const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
|
||||
if (provisioningCancelled(job.id)) return;
|
||||
if (rebalance && (health.state !== 'running' || health.healthy !== true)) {
|
||||
hold('The source lobby is no longer healthy and running. Waiting for current health evidence.'); return;
|
||||
}
|
||||
if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
|
||||
const occupancy = await probePlayerEvidence(job.server);
|
||||
if (provisioningCancelled(job.id)) return;
|
||||
if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; }
|
||||
}
|
||||
db().transaction(() => {
|
||||
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
|
||||
if (!state || !['waiting','queued','interrupted'].includes(state.state) || hostMigration(job.id)?.remove_operation_id) return;
|
||||
const current = nativeInstance(job.server);
|
||||
if (current?.id !== instance.id || current.generation !== instance.generation
|
||||
|| getHost(instance.host_id)?.mode !== (rebalance ? 'active' : 'draining')) return;
|
||||
const blocker = rebalanceAdmission(job.id) ?? eligibility() ?? activeBetaRemovalBlocker(job.server);
|
||||
if (blocker) { hold(blocker); return; }
|
||||
if (rebalance && !moveRecipeMatches(job.server, { format: 1, ...recipe! })) {
|
||||
hold('The deployed files changed; stop this run and generate a fresh rebalance preview.'); return;
|
||||
}
|
||||
if (rebalance) {
|
||||
const startups = db().prepare(`SELECT COUNT(*) AS n FROM (
|
||||
SELECT id FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
|
||||
AND action IN ('create','rebuild') AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})
|
||||
UNION SELECT value FROM json_each(?) WHERE value!=?)`).get(job.id, JSON.stringify([...preparingJobs]), job.id) as { n: number };
|
||||
if (startups.n >= fleetSettings().createConcurrency) { hold('Waiting for startup capacity before moving this lobby.'); return; }
|
||||
}
|
||||
const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
|
||||
if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
|
||||
const { host, port } = choosePlacement(options);
|
||||
const capacityBlocker = concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? concurrencyBlocker(job.id, 'rebuild', host.id);
|
||||
if (capacityBlocker) { hold(capacityBlocker); return; }
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
|
||||
const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
|
||||
WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?`)
|
||||
.get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number }).n;
|
||||
const operation = queueHostOperation(instance.host_id, 'instance.remove', {}, {
|
||||
instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:migrate-remove:${attempt}`
|
||||
});
|
||||
db().prepare('UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?')
|
||||
.run(host.id, port, host.bindAddress, operation.id, job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
|
||||
deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { phase: 'container-action', blocker: null });
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { nativeInstance, runtimePasswordForServer } from '../backends/native.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { listServerScripts } from '../fleet/inventory.js';
|
||||
import { recipeStorage, recipeSummaryForDeployment, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
|
||||
|
||||
/** Preserve the recorded deployed inventory, including changes made after launch. */
|
||||
export function captureMoveRecipe(server: string): LaunchRecipePlan {
|
||||
const instance = nativeInstance(server);
|
||||
const row = getServer(server);
|
||||
if (!instance || !row?.map_slug) throw new Error('A live native instance and saved map are required before moving.');
|
||||
if (!runtimePasswordForServer(server)) throw new Error('The deployed runtime credentials are unavailable.');
|
||||
const launch = db().prepare(`SELECT d.id FROM server_deployments d
|
||||
JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
JOIN server_launch_recipes r ON r.deployment_id=d.id
|
||||
WHERE n.instance_id=? AND d.state IN ('done','warning') AND d.server_deleted_at IS NULL
|
||||
ORDER BY d.rowid DESC LIMIT 1`).get(instance.id) as { id: string } | undefined;
|
||||
const previous = launch && recipeSummaryForDeployment(launch.id)?.recipe;
|
||||
if (!previous || previous.game !== row.game || previous.mapSlug !== row.map_slug) {
|
||||
throw new Error('The deployed launch recipe is unavailable or differs from the saved map.');
|
||||
}
|
||||
const inventory = listServerScripts(server);
|
||||
if (!inventory.length) throw new Error('The deployed file inventory is unavailable.');
|
||||
const files = inventory.map(file => {
|
||||
const before = previous.files.find(item => item.path === file.rel_path);
|
||||
const version = file.script_version_id === null ? undefined : db().prepare(`SELECT v.version,v.kind,s.name
|
||||
FROM script_versions v JOIN scripts s ON s.id=v.script_id WHERE v.id=?`).get(file.script_version_id) as
|
||||
{ version: number; kind: string; name: string } | undefined;
|
||||
const kind = version?.kind === 'baseline' ? 'config' : version?.kind ?? before?.kind;
|
||||
if (!['script', 'plugin', 'config'].includes(kind ?? '')) throw new Error(`The deployed file type is unavailable for ${file.rel_path}.`);
|
||||
if (!db().prepare('SELECT 1 FROM file_blobs WHERE sha256=? AND size=?').get(file.sha256, file.size)) {
|
||||
throw new Error(`The deployed bytes are unavailable for ${file.rel_path}.`);
|
||||
}
|
||||
return { name: version?.name ?? before?.name ?? file.rel_path, kind: kind as 'script' | 'plugin' | 'config',
|
||||
path: file.rel_path, versionId: file.script_version_id, version: version?.version ?? null,
|
||||
sha256: file.sha256, size: file.size };
|
||||
});
|
||||
if (!files.some(file => file.path === 'electric_cherry_map.cfg' && file.kind === 'config')) {
|
||||
throw new Error('The deployed map configuration is unavailable.');
|
||||
}
|
||||
return { format: 1, game: previous.game, mapSlug: previous.mapSlug, mapName: previous.mapName,
|
||||
expectedMap: previous.expectedMap, configFile: previous.configFile,
|
||||
configSource: 'Preserved deployed files', capturedAt: new Date().toISOString(), image: instance.image,
|
||||
native: { libraryId: instance.library_id, image: instance.image }, files };
|
||||
}
|
||||
|
||||
/** Immutable hash references keep deployed bytes alive without decoding plugins on the controller. */
|
||||
export function moveRecipeStorage(recipe: LaunchRecipePlan): ReturnType<typeof recipeStorage> {
|
||||
const metadata = { ...recipe, files: recipe.files.map(({ content: _content, ...file }) => file) };
|
||||
const stored = recipeStorage(metadata);
|
||||
return { recipeSummary: stored.recipeSummary,
|
||||
recipe: JSON.stringify({ ...metadata, files: metadata.files.map(file => ({ ...file, contentHash: file.sha256 })) }) };
|
||||
}
|
||||
|
||||
export function moveRecipeMatches(server: string, recipe: LaunchRecipePlan): boolean {
|
||||
const inventory = listServerScripts(server);
|
||||
return inventory.length === recipe.files.length && inventory.every(file => recipe.files.some(saved =>
|
||||
saved.path === file.rel_path && saved.sha256 === file.sha256 && saved.size === file.size));
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import { assertScalingOwnership } from '../autoscaling/ownership.js';
|
||||
import { removalAuthorization } from '../autoscaling/authorization.js';
|
||||
import { scalingIntent } from '../autoscaling/admission.js';
|
||||
import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js';
|
||||
import { concurrencyBlocker, preparingJobs } from './concurrency.js';
|
||||
import { concurrencyBlocker, preparingJobs, committedRebalanceStartup } from './concurrency.js';
|
||||
import { isIP } from 'node:net';
|
||||
import { setImmediate as yieldToController } from 'node:timers/promises';
|
||||
import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
|
||||
@@ -15,7 +15,7 @@ import { config, relDestForKind, serverDir } from '../config.js';
|
||||
import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
|
||||
import { getHost, listHosts, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
|
||||
import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
|
||||
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
|
||||
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeSummaryForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
|
||||
import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
|
||||
import { getServerKey, reserveServerKey, releaseServerKey, availableServerKeyCount } from '../fleet/server-keys.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
@@ -36,7 +36,8 @@ import type { LaunchPreview } from '../../deployment-plan.js';
|
||||
import type { ProvisioningAction } from '../../provisioning.js';
|
||||
import { hostAgentUpdatesEnabled } from '../../hosts.js';
|
||||
import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
|
||||
import { hostMigration, processMigrationRemoval, reconcileHostDrains } from './host-drain.js';
|
||||
import { reconcileHostDrains } from './host-drain.js';
|
||||
import { hostMigration, isRebalanceMigration, processMigrationRemoval } from './migrations.js';
|
||||
import { safetyOverride, safetyOverrideCapability } from './safety-override.js';
|
||||
|
||||
export type NativeSelection = import('../../hosts.js').NativePlacement;
|
||||
@@ -390,7 +391,12 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
|
||||
const activeJobs = new Map<string, Promise<void>>();
|
||||
// Preparation owns startup capacity until its command is committed or discarded.
|
||||
|
||||
function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
|
||||
function otherPreparations(id: string): number {
|
||||
if (!preparingJobs.size) return 0;
|
||||
const reserved = new Set((db().prepare(`SELECT id FROM server_deployments WHERE id IN (SELECT value FROM json_each(?))
|
||||
AND ${committedRebalanceStartup}`).all(JSON.stringify([...preparingJobs])) as { id: string }[]).map(row => row.id));
|
||||
return [...preparingJobs].filter(other => other !== id && !reserved.has(other)).length;
|
||||
}
|
||||
|
||||
let launchCommit: Promise<void> = Promise.resolve();
|
||||
function commitPreparedLaunch(commit: () => void): Promise<void> {
|
||||
@@ -469,7 +475,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
}
|
||||
const settings = fleetSettings();
|
||||
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
|
||||
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
|
||||
AND action IN ('create','rebuild') AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})`).get(job.id) as { n: number };
|
||||
if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
|
||||
const capacityBlocker = concurrencyBlocker(job.id, job.action);
|
||||
if (capacityBlocker) { hold(job, capacityBlocker); return; }
|
||||
@@ -645,7 +651,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
if (capacityBlocker) { hold(job, capacityBlocker); return; }
|
||||
const settings = fleetSettings();
|
||||
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
|
||||
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
|
||||
AND action IN (${actions.map(() => '?').join(',')}) AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})`).get(job.id, ...actions) as { n: number }).n;
|
||||
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= settings.createConcurrency) {
|
||||
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue',
|
||||
blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
|
||||
@@ -694,13 +700,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
preparingJobs.add(job.id);
|
||||
preparedKey = getServerKey(job.server) ?? null;
|
||||
if (!preparedKey) throw new Error('The saved server key was removed.');
|
||||
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
|
||||
const frozenRecipeId = migration && isRebalanceMigration(job.id) ? job.id : undefined;
|
||||
const prior = (frozenRecipeId ? { format: 1, ...recipeSummaryForDeployment(job.id)!.recipe }
|
||||
: recipeForDeployment(job.id)!.recipe) as NativeRecipe;
|
||||
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare',
|
||||
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
|
||||
preparedPassword = config.rcon.password || randomBytes(32).toString('hex');
|
||||
prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
|
||||
instance, serverKey: preparedKey, rconPassword: preparedPassword,
|
||||
rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety });
|
||||
rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety, frozenRecipeId });
|
||||
if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
|
||||
}
|
||||
const commit = db().transaction(() => {
|
||||
@@ -731,8 +739,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
if (job.action === 'destroy' && count(['destroy']) >= fleetSettings().destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
|
||||
if (prepared) {
|
||||
// Compare the same runtime config bytes while rechecking current promoted files.
|
||||
if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe,
|
||||
captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug, preparedPassword))) {
|
||||
const expected = migration && isRebalanceMigration(job.id)
|
||||
? { format: 1 as const, ...recipeSummaryForDeployment(job.id)!.recipe }
|
||||
: captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug, preparedPassword);
|
||||
if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe, expected)) {
|
||||
hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
|
||||
}
|
||||
assertReadyLibrary(instance!.host_id, instance!.library_id);
|
||||
|
||||
Reference in New Issue
Block a user