josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add safe manual instance rebalancing

Browse Source
This commit is contained in:
josh
2026-10-08 12:37:56 -04:00
parent 26de43639f
commit 012cd51a66
45 changed files with 2006 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/native.ts
+20 -10
View File
@@ -2,7 +2,7 @@ import { assertScalingOwnership } from '../autoscaling/ownership.js';
import { removalAuthorization } from '../autoscaling/authorization.js';
import { scalingIntent } from '../autoscaling/admission.js';
import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js';
import { concurrencyBlocker, preparingJobs } from './concurrency.js';
import { concurrencyBlocker, preparingJobs, committedRebalanceStartup } from './concurrency.js';
import { isIP } from 'node:net';
import { setImmediate as yieldToController } from 'node:timers/promises';
import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
@@ -15,7 +15,7 @@ import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
import { getHost, listHosts, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeSummaryForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
import { getServerKey, reserveServerKey, releaseServerKey, availableServerKeyCount } from '../fleet/server-keys.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
@@ -36,7 +36,8 @@ import type { LaunchPreview } from '../../deployment-plan.js';
import type { ProvisioningAction } from '../../provisioning.js';
import { hostAgentUpdatesEnabled } from '../../hosts.js';
import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
import { hostMigration, processMigrationRemoval, reconcileHostDrains } from './host-drain.js';
import { reconcileHostDrains } from './host-drain.js';
import { hostMigration, isRebalanceMigration, processMigrationRemoval } from './migrations.js';
import { safetyOverride, safetyOverrideCapability } from './safety-override.js';
export type NativeSelection = import('../../hosts.js').NativePlacement;
@@ -390,7 +391,12 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
const activeJobs = new Map<string, Promise<void>>();
// Preparation owns startup capacity until its command is committed or discarded.
function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
function otherPreparations(id: string): number {
if (!preparingJobs.size) return 0;
const reserved = new Set((db().prepare(`SELECT id FROM server_deployments WHERE id IN (SELECT value FROM json_each(?))
AND ${committedRebalanceStartup}`).all(JSON.stringify([...preparingJobs])) as { id: string }[]).map(row => row.id));
return [...preparingJobs].filter(other => other !== id && !reserved.has(other)).length;
}
let launchCommit: Promise<void> = Promise.resolve();
function commitPreparedLaunch(commit: () => void): Promise<void> {
@@ -469,7 +475,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
}
const settings = fleetSettings();
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
AND action IN ('create','rebuild') AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})`).get(job.id) as { n: number };
if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
const capacityBlocker = concurrencyBlocker(job.id, job.action);
if (capacityBlocker) { hold(job, capacityBlocker); return; }
@@ -645,7 +651,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (capacityBlocker) { hold(job, capacityBlocker); return; }
const settings = fleetSettings();
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
AND action IN (${actions.map(() => '?').join(',')}) AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})`).get(job.id, ...actions) as { n: number }).n;
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= settings.createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue',
blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
@@ -694,13 +700,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
preparingJobs.add(job.id);
preparedKey = getServerKey(job.server) ?? null;
if (!preparedKey) throw new Error('The saved server key was removed.');
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
const frozenRecipeId = migration && isRebalanceMigration(job.id) ? job.id : undefined;
const prior = (frozenRecipeId ? { format: 1, ...recipeSummaryForDeployment(job.id)!.recipe }
: recipeForDeployment(job.id)!.recipe) as NativeRecipe;
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
preparedPassword = config.rcon.password || randomBytes(32).toString('hex');
prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
instance, serverKey: preparedKey, rconPassword: preparedPassword,
rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety });
rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety, frozenRecipeId });
if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
}
const commit = db().transaction(() => {
@@ -731,8 +739,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (job.action === 'destroy' && count(['destroy']) >= fleetSettings().destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
if (prepared) {
// Compare the same runtime config bytes while rechecking current promoted files.
if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe,
captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug, preparedPassword))) {
const expected = migration && isRebalanceMigration(job.id)
? { format: 1 as const, ...recipeSummaryForDeployment(job.id)!.recipe }
: captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug, preparedPassword);
if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe, expected)) {
hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
}
assertReadyLibrary(instance!.host_id, instance!.library_id);