Use the 95th percentile for startup capacity

docs/guides/native-hosts.md

@@ -255,8 +255,8 @@ startup samples require observing the first five minutes of a container's run.
 
 The controller defaults to retaining per-instance minute peaks for 24 hours and caches learned
 CPU/memory allowances. The first five minutes after a container starts count as
-startup; later samples train the running allowance. Startup uses the observed
+startup; later samples train the running allowance. Both phases use the
-maximum and running uses the 95th percentile of minute peaks, with **no per-server
+95th percentile of minute peaks, with **no per-server
 headroom**. Each calculation considers up to the latest 10,000 minute peaks.
 Host-local evidence is preferred to account for different hardware; new hosts
 can borrow the fleet's learned allowance. Startup allowances are never lower
@@ -276,7 +276,8 @@ if observations exceed them. They are replaced by measured allowances once the
 evidence threshold is met; a tiny 0.01-core floor prevents zero-CPU admission.
 Models expire after 24 hours without measurements. Recalculation happens during
 heartbeat ingestion, at most once per minute per host/fleet phase; new peaks
-raise the cache immediately. Launch admission reads only the cached model.
+raise the cache immediately. Existing startup estimates refresh when new startup
+measurements arrive. Launch admission reads only the cached model.
 
 ### Admission and placement
 
@@ -285,7 +286,7 @@ These allowances are scheduling estimates, not Docker limits. Collection preview
 estimate how many new servers fit at their running allowance. Jobs may still be
 queued when that estimate is insufficient. Current host CPU and
 available memory account for existing containers and other processes. Observed
-containers are not topped up to historical startup or running peaks, even while
+containers are not topped up to historical startup or running allowances, even while
 awaiting health verification. Established instances are covered by host usage
 when individual container measurements are unavailable.
 
@@ -294,7 +295,7 @@ Unplaced queued launches wait their turn without holding physical resources.
 Any previously assigned reservations remain protected. Submitted or uncertain launches missing a matching-generation
 measurement still consume a full startup allowance until telemetry accounts for
 them. This lets collections start as capacity
-becomes available, without stacking every queued server's startup peak.
+becomes available, without stacking every queued server's startup allowance.
 Capacity errors report projected usage, usable capacity and reservations.
 
 When startup capacity is available, provisioning checks each eligible host’s

src/lib/server/fleet/launch-recipes.ts

@@ -76,12 +76,12 @@ export function captureRecipe(game: ZombieGame, mapSlug: string, rconPassword?:
 }
 
 /** Preview and admission keep file references; execution captures the current bytes. */
-export function captureRecipePlan(game: ZombieGame, mapSlug: string): LaunchRecipePlan {
+export function captureRecipePlan(game: ZombieGame, mapSlug: string, rconPassword?: string): LaunchRecipePlan {
-    return buildRecipe(game, mapSlug, false);
+    return buildRecipe(game, mapSlug, false, rconPassword);
 }
 
 function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: true, rconPassword?: string): LaunchRecipe;
-function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: false): LaunchRecipePlan;
+function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: false, rconPassword?: string): LaunchRecipePlan;
 function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean, rconPassword?: string): LaunchRecipePlan {
     const map = zombieMapBySlug(game, mapSlug);
     if (!map) throw new Error('Choose a supported game and map.');

src/lib/server/hosts/calibration.ts

@@ -82,7 +82,7 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory
                 WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
                 .all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
                 { cpu_cores: number; memory_bytes: number; bucket: string }[];
-            const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
+            const percentile = (values: number[]) => values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1];
             const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu);
             const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory));
             handle.prepare(`INSERT INTO ec_resource_allowances

src/lib/server/provisioning/native.ts

@@ -683,6 +683,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
     let prepared: PreparedNativeLaunch | undefined;
     let preparedKey: string | null = null;
+    let preparedPassword: string | undefined;
     try {
         if (job.action === 'create' || job.action === 'rebuild') {
             if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
@@ -696,8 +697,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
             deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare',
                 checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
+            preparedPassword = config.rcon.password || randomBytes(32).toString('hex');
             prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
-                instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
+                instance, serverKey: preparedKey, rconPassword: preparedPassword,
                 rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety });
             if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
         }
@@ -728,8 +730,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             }
             if (job.action === 'destroy' && count(['destroy']) >= fleetSettings().destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
             if (prepared) {
+                // Compare the same runtime config bytes while rechecking current promoted files.
                 if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe,
-                    captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug))) {
+                    captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug, preparedPassword))) {
                     hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
                 }
                 assertReadyLibrary(instance!.host_id, instance!.library_id);

tests/unit/host-capacity-calibration.test.ts

@@ -196,13 +196,29 @@ describe('measured instance capacity', () => {
         expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1);
         expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
     });
-    it('uses a running percentile and startup peaks without padding, keeping admission reads cached', async () => {
+    it.each(['startup', 'running'] as const)('uses the %s 95th percentile for CPU and memory, keeping admission reads cached', async phase => {
-        for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 });
+        await learned();
+        // Repeated starts or running observations include isolated CPU and
+        // memory outliers in different minutes; neither should set the allowance.
+        const uptimeSeconds = phase === 'startup' ? 60 : 600;
+        for (let minute = 25; minute < 125; minute++) await report(minute, {
+            uptimeSeconds, cpuCores: minute === 25 ? 2 : 0.4, memoryBytes: minute === 26 ? 3 * GiB : GiB / 2
+        });
         const { db } = await import('../../src/lib/server/db/index.js');
         const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
         const prepare = vi.spyOn(db(), 'prepare');
-        expect(capacityCalibration(hostId).running.cpuCores).toBe(0.1);
+        expect(capacityCalibration(hostId)[phase]).toMatchObject({ source: 'host', cpuCores: 0.4, memoryBytes: GiB / 2 });
+        expect(capacityCalibration('new-host')[phase]).toMatchObject({ source: 'fleet', cpuCores: 0.4, memoryBytes: GiB / 2 });
         expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false);
         prepare.mockRestore();
+
+        // Both phases still raise immediately for fresh spikes, hold that value
+        // within the minute, then return to P95 on the next recalculation.
+        await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB });
+        expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB });
+        await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
+        expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB });
+        await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
+        expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 0.4, memoryBytes: GiB / 2 });
     });
 });

tests/unit/native-launch-preparation.test.ts

@@ -179,17 +179,23 @@ describe('asynchronous native launch preparation', () => {
         expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(job.deploymentId)).toEqual({ state: 'cancelled' });
     });
 
-    it('discards prepared bytes when the promoted version changes before dispatch', async () => {
+    it.each(['baseline', 'config', 'script', 'plugin'] as const)('discards prepared bytes when the promoted %s changes before dispatch', async kind => {
         const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
         const { db } = await import('../../src/lib/server/db/index.js');
-        const { script, version } = createScript({ name: 'Launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll',
+        const baseline = createScript({ name: 'Launch baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5',
-            scopeType: 'game', scopeValue: 't5', content: Buffer.from('before promotion') });
+            content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') });
+        db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(baseline.version.id, baseline.script.id);
+        const isConfig = kind === 'baseline' || kind === 'config';
+        const { script, version } = kind === 'baseline' ? baseline : createScript({ name: `Launch ${kind}`, kind,
+            targetRelPath: kind === 'script' ? 'fixture.gsc' : kind === 'plugin' ? 'fixture.dll' : undefined,
+            scopeType: kind === 'config' ? 'map' : 'game', scopeValue: kind === 'config' ? 'zombie_theater' : 't5',
+            content: Buffer.from(isConfig ? 'set sv_voice "1"\n' : 'before promotion') });
         db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
         const job = await queue();
         const paused = await pausePreparation();
         const pass = runPass();
         await paused.ready;
-        const content = Buffer.from('newly promoted plugin');
+        const content = Buffer.from(isConfig ? 'set rcon_password "password"\nset sv_voice "0"\n' : 'newly promoted file');
         const latest = uploadNewVersion(script.id, content);
         db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(latest.id, script.id);
         paused.release();
@@ -200,7 +206,30 @@ describe('asynchronous native launch preparation', () => {
         expect(operation).toBeDefined();
         const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
         const payload = getHostOperation(operation!.id)!.payload as { files: { contentBase64: string; sha256: string }[] };
-        expect(payload.files).toContainEqual(expect.objectContaining({ contentBase64: content.toString('base64'), sha256: latest.sha256 }));
+        if (isConfig) {
+            const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
+            const expected = getPromotedServerConfig('t5', 'zombie_theater', 'theater', 'test-password')!;
+            expect(payload.files).toContainEqual(expect.objectContaining({ contentBase64: expected.toString('base64') }));
+            expect(expected.toString()).toContain('set sv_voice "0"');
+        } else expect(payload.files).toContainEqual(expect.objectContaining({ contentBase64: content.toString('base64'), sha256: latest.sha256 }));
+    });
+
+    it('discards prepared bytes when the server key changes before dispatch', async () => {
+        const job = await queue();
+        const paused = await pausePreparation();
+        const pass = runPass();
+        await paused.ready;
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare(`UPDATE server_keys SET pluto_key='changed-key' WHERE id IN
+            (SELECT key_id FROM server_key_leases WHERE server_id=? AND is_current=1)`).run(job.name);
+        paused.release();
+        await pass;
+        expect(await operationFor(job.deploymentId)).toBeUndefined();
+        await runPass();
+        const operation = await operationFor(job.deploymentId);
+        expect(operation).toBeDefined();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        expect(getHostOperation(operation!.id)!.payload).toMatchObject({ env: { PLUTO_SERVER_KEY: 'changed-key' } });
     });
 
     it('prepares only one launch while the single startup slot is reserved', async () => {

tests/unit/native-provisioning.test.ts

@@ -84,6 +84,40 @@ async function observations(server: string) {
     return [serverObservation(row), observationReader()(row), new MonitoringSnapshot().observe(row)];
 }
 
+describe('runtime configuration at dispatch', () => {
+    it.each([
+        ['t5', 'shared'], ['t5', 'generated'], ['t6', 'shared'], ['t6', 'generated']
+    ] as const)('dispatches %s with a promoted baseline and %s RCON password', async (game, source) => {
+        const { config, configFileForGame } = await import('../../src/lib/server/config.js');
+        if (source === 'generated') Object.assign(config.rcon, { password: '' });
+        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { script, version } = createScript({ name: 'Launch baseline', kind: 'baseline', scopeType: 'game', scopeValue: game,
+            content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') });
+        db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
+
+        const job = await queue(game);
+        await advance();
+        const operation = await operationFor(job.deploymentId);
+        expect(operation).toMatchObject({ kind: 'instance.create', state: 'pending' });
+        const payload = operation!.payload as { rconPassword: string; files: { path: string; contentBase64: string; sha256: string }[] };
+        expect(payload.rconPassword).toEqual(source === 'shared' ? 'test-password' : expect.stringMatching(/^[a-f0-9]{64}$/));
+        const configFile = configFileForGame(game);
+        const file = payload.files.find(file => file.path === `configs/${configFile}`)!;
+        const content = Buffer.from(file.contentBase64, 'base64');
+        expect(content.toString()).toContain(`set rcon_password "${payload.rconPassword}"`);
+        expect(content.toString()).not.toContain('set rcon_password "password"');
+        const hash = createHash('sha256').update(content).digest('hex');
+        expect(file.sha256).toBe(hash);
+        const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
+        expect(getServerScript(job.name, configFile)).toMatchObject({ sha256: hash, size: content.length, source: 'native-create' });
+        const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
+        expect(recipeForDeployment(job.deploymentId)!.recipe.files).toContainEqual(expect.objectContaining({
+            path: configFile, sha256: hash, size: content.length, content: content.toString('base64')
+        }));
+    });
+});
+
 describe('provisioning health handoff', () => {
     it.each(['create', 'start', 'restart', 'rebuild'] as const)('%s publishes verified game health before completion becomes visible', async action => {
         const job = await queue(); await advance();