Pool Plutonium keys by game for launches and autoscaling

src/lib/server/autoscaling/index.ts

@@ -2,7 +2,7 @@ import { createHash, randomUUID } from 'node:crypto';
 import { db } from '../db/index.js';
 import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js';
 import { emptyObservation, planScaling } from './planner.js';
-import { zombieMaps, parseServerName } from '../../map-aliases.js';
+import { parseServerName } from '../../map-aliases.js';
 import { scalingSnapshot } from './snapshot.js';
 import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
 import { cancelCreation } from '../provisioning/cancel-creation.js';
@@ -30,13 +30,6 @@ export function previewScaling(input: unknown, id?: string, revision?: number):
         nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
     const snapshot = scalingSnapshot(policy);
     const blockers: string[] = [];
-    // Creating any policy requires full coverage; existing policies must always be pausable.
-    if (!existing || policy.mode !== 'off') {
-        for (const map of zombieMaps(policy.game).filter(m => policy.mapSlug === '*' || m.slug === policy.mapSlug)) {
-            const count = snapshot.keyedNames.filter(name => parseServerName(name)?.mapSlug === map.slug).length;
-            if (count < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities for ${map.friendly} before saving this policy (${count} saved; ${policy.maximum - count} more required).`);
-        }
-    }
     if (liveScaling(policy.mode)) {
         const owner = scalingOwner(policy.game, policy.mapSlug);
         if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`);
@@ -51,7 +44,7 @@ export function previewScaling(input: unknown, id?: string, revision?: number):
     }
     const decision = planScaling(policy, snapshot, Date.now());
     const token = createHash('sha256').update(JSON.stringify({ settings, revision: policy.revision, globalPaused: scalingGloballyPaused(),
-        members: snapshot.members.map(m => [m.name, m.identity, m.state]), keys: snapshot.keyedNames.sort(), reservations: snapshot.reservedNames.sort(), blockers: snapshot.blockers, decision })).digest('hex');
+        members: snapshot.members.map(m => [m.name, m.identity, m.state]), keyPool: snapshot.keyPool, reservations: snapshot.reservedNames.sort(), blockers: snapshot.blockers, decision })).digest('hex');
     return { policy, snapshot, decision, token, blockers };
 }
 
@@ -154,7 +147,7 @@ export async function runAutoscaling(): Promise<boolean> {
                     ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation));
             }
             // Evidence includes provenance and timestamps but never keys or player rosters.
-            const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyCount: snapshot.keyedNames.length, blockers: snapshot.blockers });
+            const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyPool: snapshot.keyPool, blockers: snapshot.blockers });
             const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
             const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
             const execute = liveScaling(policy.mode) && (decision.action === 'create' || policy.mode === 'create-destroy' && decision.action === 'destroy');

src/lib/server/autoscaling/planner.ts

@@ -21,7 +21,7 @@ export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, no
             const matches = (name: string) => (snapshot.members.find(m => m.name === name)?.mapSlug ?? parseServerName(name)?.mapSlug) === map.slug;
             return planScaling({ ...policy, mapSlug: map.slug }, {
                 ...snapshot, members: snapshot.members.filter(m => matches(m.name)),
-                reservedNames: snapshot.reservedNames.filter(matches), keyedNames: snapshot.keyedNames.filter(matches)
+                reservedNames: snapshot.reservedNames.filter(matches)
             }, now);
         });
         // Serialize game-wide work and prefer replenishing any map before trimming another.
@@ -50,8 +50,12 @@ export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, no
     if (policy.nextActionAt > now) return result('wait', 'Action cooldown is active.');
     if (occupied + empty < desired) {
         if (snapshot.reservedNames.length >= policy.maximum) return result('wait', 'Maximum lobby reservations reached.');
-        const available = snapshot.keyedNames.filter(name => !snapshot.reservedNames.includes(name)).sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
-        return available.length ? result('create', 'Replenish empty lobby capacity.', available[0]) : result('wait', 'No unused lobby identity has a saved key.');
+        if (!snapshot.keyPool.available) return result('wait', snapshot.keyPool.total
+            ? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.`
+            : `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`);
+        const available = Array.from({ length: 20 }, (_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`)
+            .find(name => !snapshot.reservedNames.includes(name));
+        return available ? result('create', 'Replenish empty lobby capacity.', available) : result('wait', 'No unused lobby identity is available.');
     }
     if (occupied + empty > desired && empty > policy.spares && occupied + empty > policy.minimum) {
         const candidates = members.filter(m => m.state === 'empty' && m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000)

src/lib/server/autoscaling/snapshot.ts

@@ -1,5 +1,5 @@
 import { db } from '../db/index.js';
-import { validServerId } from '../fleet/server-keys.js';
+import { serverKeyPool } from '../fleet/server-keys.js';
 import { listServers, type ServerRow } from '../fleet/servers.js';
 import { observationReader } from '../fleet/status.js';
 import { parseServerName, zombieMaps } from '../../map-aliases.js';
@@ -73,6 +73,8 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
         reservedNames.add(row.name);
         if (!members.some(m => m.name === row.name)) blockers.push(`Unresolved native instance ${row.name}.`);
     }
-    const keyedNames = (handle.prepare("SELECT server_id AS name FROM server_keys WHERE length(trim(pluto_key))>0").all() as { name: string }[]).filter(row => validServerId(row.name) && matches(row)).map(r => r.name);
-    return { members, reservedNames: [...reservedNames], keyedNames, blockers };
+    for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as { name: string }[]) if (matches(row)) reservedNames.add(row.name);
+    const ownServer = exceptOperation ? (handle.prepare('SELECT server FROM server_deployments WHERE id=?').get(exceptOperation) as { server: string } | undefined)?.server : undefined;
+    const { total, available } = serverKeyPool(policy.game, ownServer);
+    return { members, reservedNames: [...reservedNames], keyPool: { total, available }, blockers };
 }

src/lib/server/db/index.ts

@@ -10,6 +10,7 @@ import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js';
 import { initializeFleetSettings } from './fleet-settings.js';
 import { migrateServerHistory } from './server-history.js';
 import { migrateNativeInstances } from '../backends/schema.js';
+import { migrateServerKeys } from './server-keys.js';
 import { ensureAutoscalingSchema } from '../autoscaling/schema.js';
 import { installPerformanceIndexes } from './performance-indexes.js';
 import { migrateFileBlobs } from './file-blobs.js';
@@ -397,6 +398,7 @@ export function db(): Database.Database {
         backfillLeaderboard(handle);
         initializeFleetSettings(handle);
         migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
+        migrationStep('per-game server key pools', () => migrateServerKeys(handle));
         migrationStep('autoscaling schema', () => ensureAutoscalingSchema(handle));
         // File-blob migration removes the legacy content column altogether.
         // Reordering it first would unnecessarily copy every artifact twice.

src/lib/server/db/schema.sql

@@ -433,11 +433,15 @@ CREATE TABLE IF NOT EXISTS deployment_scripts (
 );
 CREATE INDEX IF NOT EXISTS idx_deployment_scripts_dep ON deployment_scripts(deployment_id);
 
--- Deliberately independent of servers: credentials survive server destruction.
+-- Credentials belong to a game and survive server destruction. Reservations
+-- are installed by the key-pool migration after native instance schema exists.
 CREATE TABLE IF NOT EXISTS server_keys (
-    server_id TEXT PRIMARY KEY,
+    id TEXT PRIMARY KEY,
+    game TEXT NOT NULL CHECK (game IN ('t5','t6')),
+    label TEXT NOT NULL DEFAULT '',
     pluto_key TEXT NOT NULL,
-    updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+    updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+    UNIQUE(game, pluto_key)
 );
 
 CREATE TABLE IF NOT EXISTS settings (

src/lib/server/db/server-keys.ts

@@ -0,0 +1,100 @@
+import { randomUUID } from 'node:crypto';
+import type Database from 'better-sqlite3';
+
+const keySchema = `CREATE TABLE IF NOT EXISTS server_keys (
+    id TEXT PRIMARY KEY,
+    game TEXT NOT NULL CHECK (game IN ('t5','t6')),
+    label TEXT NOT NULL DEFAULT '',
+    pluto_key TEXT NOT NULL,
+    updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+    UNIQUE(game, pluto_key)
+);`;
+
+const leaseSchema = `CREATE TABLE IF NOT EXISTS server_key_leases (
+    server_id TEXT NOT NULL,
+    key_id TEXT NOT NULL REFERENCES server_keys(id) ON DELETE RESTRICT,
+    is_current INTEGER NOT NULL DEFAULT 1 CHECK (is_current IN (0,1)),
+    reserved_at TEXT NOT NULL DEFAULT (datetime('now')),
+    PRIMARY KEY(server_id,key_id)
+);
+CREATE INDEX IF NOT EXISTS idx_server_key_leases_key ON server_key_leases(key_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_server_key_leases_current ON server_key_leases(server_id) WHERE is_current=1;`;
+
+/** Preserve every occupied legacy assignment, even duplicated credentials. */
+export function migrateServerKeys(handle: Database.Database): void {
+    handle.transaction(() => {
+        const legacy = (handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[])
+            .some(column => column.name === 'server_id');
+        if (legacy) handle.exec('ALTER TABLE server_keys RENAME TO legacy_server_keys');
+        handle.exec(keySchema + leaseSchema);
+        if (!legacy) return;
+        const rows = handle.prepare(`SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id`).all() as
+            { server_id: string; pluto_key: string; updated_at: string }[];
+        const occupied = handle.prepare(`SELECT 1 WHERE
+            EXISTS (SELECT 1 FROM servers WHERE name=@server AND destroyed_at IS NULL)
+            OR EXISTS (SELECT 1 FROM native_instances WHERE server=@server AND retired_at IS NULL)
+            OR EXISTS (SELECT 1 FROM server_deployments WHERE server=@server AND server_deleted_at IS NULL
+                AND action IN ('create','rebuild') AND state NOT IN ('done','warning','cancelled'))`);
+        const find = handle.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?');
+        const serverGame = handle.prepare("SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')");
+        const insert = handle.prepare('INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)');
+        const lease = handle.prepare('INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)');
+        const assignments = new Map<string, string[]>();
+        function save(game: string, credential: string, label: string, updatedAt: string): string {
+            const existing = (find.get(game, credential) as { id: string } | undefined)?.id;
+            if (existing) return existing;
+            const id = randomUUID();
+            insert.run(id, game, label, credential, updatedAt);
+            return id;
+        }
+        for (const row of rows) {
+            const game = /^(t5|t6)_/.exec(row.server_id)?.[1]
+                ?? (serverGame.get(row.server_id) as { game: string } | undefined)?.game;
+            if (!game) throw new Error(`Cannot determine the game for saved server key ${row.server_id}`);
+            const id = save(game, row.pluto_key, row.server_id, row.updated_at);
+            if (occupied.get({ server: row.server_id })) assignments.set(row.server_id, [id]);
+        }
+        // The old form could overwrite a saved key without changing the running
+        // container. Durable create/rebuild payloads are stronger evidence of
+        // its actual credential. Keep ambiguous delivered attempts as additional
+        // holds until the instance is destroyed; never free a possibly live key.
+        const hasTable = (name: string) => !!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name);
+        if (hasTable('ec_host_operations')) {
+            const payload = hasTable('ec_host_operation_payloads')
+                ? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)' : 'o.payload';
+            const command = `CASE WHEN json_valid(${payload}) THEN ${payload} ELSE '{}' END`;
+            const launched = handle.prepare(`SELECT n.server,o.state,o.updated_at,
+                    json_extract(${command},'$.env.PLUTO_SERVER_KEY') AS credential,
+                    json_extract(${command},'$.game') AS game,
+                    json_extract(${command},'$.env.PLUTAINER_GAME') AS game_id
+                FROM native_instances n JOIN ec_host_operations o ON o.instance_id=n.id AND o.host_id=n.host_id
+                WHERE n.retired_at IS NULL AND o.kind IN ('instance.create','instance.rebuild') AND o.state!='failed'
+                    AND (o.generation=n.generation OR json_extract(${command},'$.nextGeneration')=n.generation)
+                ORDER BY o.rowid DESC`).all() as
+                { server: string; state: string; updated_at: string; credential: unknown; game: unknown; game_id: unknown }[];
+            const recovered = new Map<string, string[]>();
+            const complete = new Set<string>();
+            for (const row of launched) {
+                if (complete.has(row.server) || typeof row.credential !== 'string' || !row.credential) continue;
+                const game = row.game === 't5' || row.game === 't6' ? row.game
+                    : row.game_id === 't5sp' ? 't5' : row.game_id === 't6zm' ? 't6'
+                    : /^(t5|t6)_/.exec(row.server)?.[1]
+                        ?? (serverGame.get(row.server) as { game: string } | undefined)?.game;
+                if (!game) throw new Error(`Cannot determine the game for launched server ${row.server}`);
+                const ids = recovered.get(row.server) ?? [];
+                const id = save(game, row.credential, row.server, row.updated_at);
+                if (!ids.includes(id)) ids.push(id);
+                recovered.set(row.server, ids);
+                if (row.state === 'completed') complete.add(row.server);
+            }
+            for (const [server, ids] of recovered) {
+                // If no completed launch receipt survives, its saved assignment
+                // remains a possible prior credential while the attempt resolves.
+                if (!complete.has(server)) for (const id of assignments.get(server) ?? []) if (!ids.includes(id)) ids.push(id);
+                assignments.set(server, ids);
+            }
+        }
+        for (const [server, ids] of assignments) ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0));
+        handle.exec('DROP TABLE legacy_server_keys');
+    }).immediate();
+}

src/lib/server/fleet/collections.ts

@@ -13,6 +13,7 @@ import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
 import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
 import { nativeDeployment, nativeInstance } from '../backends/native.js';
 import { launchAvailability } from '../hosts/availability.js';
+import { availableServerKeyCount } from './server-keys.js';
 
 function coverage(entries: BlueprintEntry[], includeHealth = true) {
     type Server = { name: string; game: string | null; map_slug: string | null; map: string | null };
@@ -72,6 +73,8 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, include
 
 function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] {
     const occupied = db().prepare(`SELECT name FROM servers UNION
+        SELECT server AS name FROM native_instances WHERE retired_at IS NULL UNION
+        SELECT server_id AS name FROM server_key_leases UNION
         SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
         WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
             OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`)
@@ -83,7 +86,7 @@ function allocate(entries: BlueprintEntry[], counts = coverage(entries)): Collec
         const taken = lobbies.get(match[1]) ?? new Set<number>();
         taken.add(Number(match[2])); lobbies.set(match[1], taken);
     }
-    const savedKeys = new Set((db().prepare("SELECT server_id FROM server_keys WHERE length(trim(pluto_key))>0").all() as { server_id: string }[]).map(row => row.server_id));
+    const availableKeys = { t5: availableServerKeyCount('t5'), t6: availableServerKeyCount('t6') };
     return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
         const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
         const taken = lobbies.get(base) ?? new Set<number>();
@@ -92,7 +95,9 @@ function allocate(entries: BlueprintEntry[], counts = coverage(entries)): Collec
         const native = entry.recipe.native;
         const port = null;
         if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
-        return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && savedKeys.has(name) };
+        const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0;
+        if (hasSavedKey) availableKeys[entry.recipe.game]--;
+        return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey };
     }));
 }
 
@@ -185,7 +190,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
         const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
         if (blockers.length) throw new Error(blockers.join(' '));
         if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) {
-            throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review each server key.');
+            throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.');
         }
         const expected = new Set(servers.map(server => server.name));
         for (const [name, value] of Object.entries(keys)) {

src/lib/server/fleet/server-keys.ts

@@ -1,35 +1,122 @@
+import { randomUUID } from 'node:crypto';
 import { db } from '../db/index.js';
-import { zombieMapBySlug } from './map-aliases.js';
 
-export function validServerId(value: unknown): value is string {
-    if (typeof value !== 'string') return false;
-    const match = /^(t5|t6)_zombie_([a-z0-9]+)_([1-9]|1[0-9]|20)$/.exec(value);
-    return !!match && !!zombieMapBySlug(match[1], match[2]);
+type KeyGame = 't5' | 't6';
+export interface ServerKey {
+    id: string;
+    game: KeyGame;
+    label: string;
+    updated_at: string;
+    assigned_servers: string[];
 }
 
-export function listServerKeys() {
-    return db().prepare(`SELECT k.server_id, k.updated_at,
-        EXISTS(SELECT 1 FROM servers s WHERE s.name = k.server_id) AS deployed
-        FROM server_keys k ORDER BY k.server_id`).all() as Array<{
-            server_id: string; updated_at: string; deployed: number;
-        }>;
+function validGame(game: unknown): asserts game is KeyGame {
+    if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 for the server key');
 }
 
-export function getServerKey(serverId: string): string | undefined {
-    return (db().prepare('SELECT pluto_key FROM server_keys WHERE server_id = ?')
-        .get(serverId) as { pluto_key: string } | undefined)?.pluto_key;
-}
-
-export function saveServerKey(serverId: unknown, key: unknown): void {
-    if (!validServerId(serverId)) throw new Error('Choose a valid T5 or T6 zombie server ID with a lobby number from 1 to 20');
+function normalizedKey(key: unknown): string {
     if (typeof key !== 'string' || !key.trim() || key.trim().length > 512 || /\s/.test(key.trim())) {
         throw new Error('Enter a server key without whitespace (maximum 512 characters)');
     }
-    db().prepare(`INSERT INTO server_keys (server_id, pluto_key) VALUES (?, ?)
-        ON CONFLICT(server_id) DO UPDATE SET pluto_key = excluded.pluto_key, updated_at = datetime('now')`)
-        .run(serverId, key.trim());
+    return key.trim();
 }
 
-export function removeServerKey(serverId: string): boolean {
-    return db().prepare('DELETE FROM server_keys WHERE server_id = ?').run(serverId).changes > 0;
+function normalizedLabel(label: unknown): string {
+    if (typeof label !== 'string' || label.trim().length > 160 || /[\r\n]/.test(label)) {
+        throw new Error('Enter a key label with at most 160 characters');
+    }
+    return label.trim();
+}
+
+export function listServerKeys(): ServerKey[] {
+    const rows = db().prepare('SELECT id,game,label,updated_at FROM server_keys ORDER BY game,label,id').all() as Omit<ServerKey, 'assigned_servers'>[];
+    const assignments = db().prepare(`SELECT DISTINCT k.id AS key_id,l.server_id FROM server_keys k
+        JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key JOIN server_key_leases l ON l.key_id=occupied.id
+        ORDER BY l.server_id`).all() as { key_id: string; server_id: string }[];
+    const byKey = new Map<string, string[]>();
+    for (const lease of assignments) {
+        const servers = byKey.get(lease.key_id) ?? [];
+        servers.push(lease.server_id);
+        byKey.set(lease.key_id, servers);
+    }
+    return rows.map(row => ({ ...row, assigned_servers: byKey.get(row.id) ?? [] }));
+}
+
+export function getServerKey(serverId: string): string | undefined {
+    return (db().prepare(`SELECT k.pluto_key FROM server_key_leases l
+        JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`).get(serverId) as { pluto_key: string } | undefined)?.pluto_key;
+}
+
+/** Saving a credential twice cannot increase available capacity. */
+export function saveServerKey(game: unknown, key: unknown, label?: unknown): string {
+    validGame(game);
+    const credential = normalizedKey(key);
+    const keyLabel = label === undefined ? undefined : normalizedLabel(label);
+    return db().transaction(() => {
+        const existing = db().prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?').get(game, credential) as { id: string } | undefined;
+        if (existing) {
+            if (keyLabel !== undefined) db().prepare("UPDATE server_keys SET label=?,updated_at=datetime('now') WHERE id=?").run(keyLabel, existing.id);
+            return existing.id;
+        }
+        const id = randomUUID();
+        db().prepare('INSERT INTO server_keys(id,game,label,pluto_key) VALUES(?,?,?,?)').run(id, game, keyLabel ?? '', credential);
+        return id;
+    }).immediate();
+}
+
+// Compare the credential as well as its row: a mistakenly imported copy in
+// another game's pool must not permit simultaneous use of the same key.
+const availableKey = `NOT EXISTS (SELECT 1 FROM server_key_leases l
+    JOIN server_keys occupied ON occupied.id=l.key_id
+    WHERE occupied.pluto_key=k.pluto_key AND (l.server_id!=@server OR l.is_current=0))`;
+
+export function serverKeyPool(game: KeyGame, serverId?: string): { total: number; available: number } {
+    validGame(game);
+    return db().prepare(`SELECT COUNT(*) AS total,
+        COALESCE(SUM(CASE WHEN ${availableKey} THEN 1 ELSE 0 END),0) AS available
+        FROM server_keys k WHERE k.game=@game`).get({ game, server: serverId ?? '' }) as { total: number; available: number };
+}
+
+export function availableServerKeyCount(game: KeyGame): number {
+    return serverKeyPool(game).available;
+}
+
+/** Called within launch admission; the write lock prevents competing launches sharing a key. */
+export function reserveServerKey(serverId: string, game: KeyGame, suppliedKey?: unknown): string {
+    validGame(game);
+    if (!serverId) throw new Error('Choose a server for the key reservation');
+    const credential = suppliedKey === undefined || suppliedKey === null || (typeof suppliedKey === 'string' && !suppliedKey.trim())
+        ? undefined : normalizedKey(suppliedKey);
+    return db().transaction(() => {
+        const existing = db().prepare(`SELECT k.game,k.pluto_key FROM server_key_leases l
+            JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`).get(serverId) as { game: KeyGame; pluto_key: string } | undefined;
+        if (existing) {
+            if (existing.game !== game) throw new Error('This server already has a key reserved for another game');
+            if (credential && credential !== existing.pluto_key) throw new Error('This server already has a reserved key; destroy it before changing its key');
+            return existing.pluto_key;
+        }
+        const requestedId = credential ? saveServerKey(game, credential) : null;
+        const key = db().prepare(`SELECT k.id,k.pluto_key FROM server_keys k WHERE k.game=@game
+            AND (@requestedId IS NULL OR k.id=@requestedId) AND ${availableKey}
+            ORDER BY k.updated_at,k.id LIMIT 1`).get({ game, server: serverId, requestedId }) as { id: string; pluto_key: string } | undefined;
+        if (!key) throw new Error(credential ? 'This server key is already reserved by another server'
+            : `No available ${game.toUpperCase()} server keys. Add a key or destroy a server to release one.`);
+        db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(serverId, key.id);
+        return key.pluto_key;
+    }).immediate();
+}
+
+/** Only after confirmed destruction or a creation cancelled before any host mutation. */
+export function releaseServerKey(serverId: string): boolean {
+    return db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(serverId).changes > 0;
+}
+
+export function removeServerKey(id: string): boolean {
+    return db().transaction(() => {
+        if (db().prepare(`SELECT 1 FROM server_keys k JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key
+            JOIN server_key_leases l ON l.key_id=occupied.id WHERE k.id=?`).get(id)) {
+            throw new Error('This key is reserved by a server. Destroy that server before removing its key.');
+        }
+        return db().prepare('DELETE FROM server_keys WHERE id=?').run(id).changes > 0;
+    }).immediate();
 }

src/lib/server/provisioning/cancel-creation.ts

@@ -1,5 +1,6 @@
 import { pauseScalingForOperation } from '../autoscaling/ownership.js';
 import { db } from '../db/index.js';
+import { releaseServerKey } from '../fleet/server-keys.js';
 import { nativeInstance } from '../backends/native.js';
 import { queueHostOperation, releaseHostPorts } from '../hosts/index.js';
 import { beginDeployment, cancelRemoval, deploymentStage, finishDeployment } from './history.js';
@@ -65,10 +66,12 @@ export function cancelCreation(id: string, triggeredBy = 'ui', pausePolicy = tru
 }
 
 export function finishCreationCancellation(cleanupId: string): void {
-    const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id FROM native_creation_cancellations c
+    const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id,d.server FROM native_creation_cancellations c
+        JOIN server_deployments d ON d.id=c.creation_id
         LEFT JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as
-        { creation_id: string; operation_id: string | null } | undefined;
+        { creation_id: string; operation_id: string | null; server: string } | undefined;
     if (!cancellation) return;
+    releaseServerKey(cancellation.server);
     const now = new Date().toISOString();
     db().prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)').run(now, cancellation.creation_id, cleanupId);
     deploymentStage(cancellation.creation_id, 'Creation cancelled; cleanup completed', { blocker: null,

src/lib/server/provisioning/instances.ts

@@ -35,7 +35,7 @@ export interface CreateServerInput {
     game?: ZombieGame;    // omitted by older clients: T5
     gameMode: GameMode;   // 'zombie' — only value supported today
     mapSlug: string;      // 'cosmodrome'
-    plutoKey?: string;    // saved for this server ID when supplied
+    plutoKey?: string;    // added to the game pool and reserved for this launch when supplied
     expectedName?: string; // optional preview guard; never selects a lobby
     previewId?: string;
     createdBy?: string;   // 'ui' by default
@@ -44,6 +44,8 @@ export interface CreateServerInput {
 export function nextLobbyNumber(baseName: string, additionalNames: string[] = []): number | null {
     const rows = db()
         .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
+            SELECT server AS name FROM native_instances WHERE retired_at IS NULL UNION
+            SELECT server_id AS name FROM server_key_leases UNION
             SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
             WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.state = 'cancelled' AND d.submitted_at IS NOT NULL) OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
         .all(`${baseName}_%`) as Array<{ name: string }>;

src/lib/server/provisioning/native.ts

@@ -17,7 +17,7 @@ import { getHost, listHosts, getHostOperationStatus, queueHostOperation, queueHo
 import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
 import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
 import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
-import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
+import { getServerKey, reserveServerKey, releaseServerKey, availableServerKeyCount } from '../fleet/server-keys.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
 import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
 import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER, cancelRemoval } from './history.js';
@@ -118,7 +118,7 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
         { label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
         { label: 'Configuration', state: 'pass', detail: recipe.configSource }
     ];
-    return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks,
+    return { id, expiresAt, name, port, hasSavedKey: !!name && availableServerKeyCount(recipe.game) > 0, recipe: summarizeRecipe(recipe), blueprintName, checks,
         ready: checks.every(check => check.state !== 'blocked') };
 }
 
@@ -163,9 +163,7 @@ export function queueNativeCreationIntent(input: { server: string; game: 't5' |
             const reserved = db().prepare(`SELECT 1 FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id WHERE j.server=?
                 AND d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health','interrupted') OR (d.state='cancelled' AND d.submitted_at IS NOT NULL))`).get(input.server);
             if ((!input.autoscalingDecisionId && input.server !== expected) || nativeInstance(input.server) || getServer(input.server) || reserved) throw new Error('Available lobby changed. Review a fresh preview.');
-            const key = input.plutoKey?.trim() || getServerKey(input.server);
-            if (!key) throw new Error(`Save a Plutonium server key for ${input.server}.`);
-            saveServerKey(input.server, key);
+            reserveServerKey(input.server, input.game, input.plutoKey);
             const recipe: NativeRecipe = { ...(input.recipe ?? captureRecipePlan(input.game, input.mapSlug)),
                 native: input.selection, image: input.selection.image! };
             const id = beginDeployment(input.server, 'create', input.createdBy ?? 'ui');
@@ -292,6 +290,7 @@ export function reconcileNativeProvisioningReceipt(id: string): void {
                 db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(removal.updatedAt, original.id);
                 releaseHostPorts(original.id);
                 db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
+                releaseServerKey(job.server);
                 db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(removal.updatedAt, id);
                 recordProgress(id, { event: { kind: 'cleanup', message: 'Cancelled migration: source removal completed; replacement creation was not started.' } });
             }
@@ -314,6 +313,7 @@ export function reconcileNativeProvisioningReceipt(id: string): void {
         db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(operation.updatedAt, instance.id);
         releaseHostPorts(instance.id);
         db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
+        releaseServerKey(job.server);
         finishCreationCancellation(job.id);
         return;
     }
@@ -539,6 +539,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance!.id);
                 releaseHostPorts(instance!.id);
                 db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
+                releaseServerKey(job.server);
                 recordProgress(job.id, { checkpoint: { key: 'cleanup', label: 'Instance retired and port reservations released' } });
                 complete(job);
                 finishCreationCancellation(job.id);