josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Pool Plutonium keys by game for launches and autoscaling

Browse Source
This commit is contained in:
josh
2026-10-07 11:24:53 -04:00
parent 56e5a2d06d
commit ec45371478
47 changed files with 956 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/db/index.ts
+2
View File
@@ -10,6 +10,7 @@ import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js';
import { initializeFleetSettings } from './fleet-settings.js';
import { migrateServerHistory } from './server-history.js';
import { migrateNativeInstances } from '../backends/schema.js';
import { migrateServerKeys } from './server-keys.js';
import { ensureAutoscalingSchema } from '../autoscaling/schema.js';
import { installPerformanceIndexes } from './performance-indexes.js';
import { migrateFileBlobs } from './file-blobs.js';
@@ -397,6 +398,7 @@ export function db(): Database.Database {
backfillLeaderboard(handle);
initializeFleetSettings(handle);
migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
migrationStep('per-game server key pools', () => migrateServerKeys(handle));
migrationStep('autoscaling schema', () => ensureAutoscalingSchema(handle));
// File-blob migration removes the legacy content column altogether.
// Reordering it first would unnecessarily copy every artifact twice.
src/lib/server/db/schema.sql
+7 -3
View File
@@ -433,11 +433,15 @@ CREATE TABLE IF NOT EXISTS deployment_scripts (
);
CREATE INDEX IF NOT EXISTS idx_deployment_scripts_dep ON deployment_scripts(deployment_id);
-- Deliberately independent of servers: credentials survive server destruction.
-- Credentials belong to a game and survive server destruction. Reservations
-- are installed by the key-pool migration after native instance schema exists.
CREATE TABLE IF NOT EXISTS server_keys (
server_id TEXT PRIMARY KEY,
id TEXT PRIMARY KEY,
game TEXT NOT NULL CHECK (game IN ('t5','t6')),
label TEXT NOT NULL DEFAULT '',
pluto_key TEXT NOT NULL,
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE(game, pluto_key)
);
CREATE TABLE IF NOT EXISTS settings (
src/lib/server/db/server-keys.ts
+100
View File
@@ -0,0 +1,100 @@
import { randomUUID } from 'node:crypto';
import type Database from 'better-sqlite3';
const keySchema = `CREATE TABLE IF NOT EXISTS server_keys (
id TEXT PRIMARY KEY,
game TEXT NOT NULL CHECK (game IN ('t5','t6')),
label TEXT NOT NULL DEFAULT '',
pluto_key TEXT NOT NULL,
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE(game, pluto_key)
);`;
const leaseSchema = `CREATE TABLE IF NOT EXISTS server_key_leases (
server_id TEXT NOT NULL,
key_id TEXT NOT NULL REFERENCES server_keys(id) ON DELETE RESTRICT,
is_current INTEGER NOT NULL DEFAULT 1 CHECK (is_current IN (0,1)),
reserved_at TEXT NOT NULL DEFAULT (datetime('now')),
PRIMARY KEY(server_id,key_id)
);
CREATE INDEX IF NOT EXISTS idx_server_key_leases_key ON server_key_leases(key_id);
CREATE UNIQUE INDEX IF NOT EXISTS idx_server_key_leases_current ON server_key_leases(server_id) WHERE is_current=1;`;
/** Preserve every occupied legacy assignment, even duplicated credentials. */
export function migrateServerKeys(handle: Database.Database): void {
handle.transaction(() => {
const legacy = (handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[])
.some(column => column.name === 'server_id');
if (legacy) handle.exec('ALTER TABLE server_keys RENAME TO legacy_server_keys');
handle.exec(keySchema + leaseSchema);
if (!legacy) return;
const rows = handle.prepare(`SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id`).all() as
{ server_id: string; pluto_key: string; updated_at: string }[];
const occupied = handle.prepare(`SELECT 1 WHERE
EXISTS (SELECT 1 FROM servers WHERE name=@server AND destroyed_at IS NULL)
OR EXISTS (SELECT 1 FROM native_instances WHERE server=@server AND retired_at IS NULL)
OR EXISTS (SELECT 1 FROM server_deployments WHERE server=@server AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state NOT IN ('done','warning','cancelled'))`);
const find = handle.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?');
const serverGame = handle.prepare("SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')");
const insert = handle.prepare('INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)');
const lease = handle.prepare('INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)');
const assignments = new Map<string, string[]>();
function save(game: string, credential: string, label: string, updatedAt: string): string {
const existing = (find.get(game, credential) as { id: string } | undefined)?.id;
if (existing) return existing;
const id = randomUUID();
insert.run(id, game, label, credential, updatedAt);
return id;
}
for (const row of rows) {
const game = /^(t5|t6)_/.exec(row.server_id)?.[1]
?? (serverGame.get(row.server_id) as { game: string } | undefined)?.game;
if (!game) throw new Error(`Cannot determine the game for saved server key ${row.server_id}`);
const id = save(game, row.pluto_key, row.server_id, row.updated_at);
if (occupied.get({ server: row.server_id })) assignments.set(row.server_id, [id]);
}
// The old form could overwrite a saved key without changing the running
// container. Durable create/rebuild payloads are stronger evidence of
// its actual credential. Keep ambiguous delivered attempts as additional
// holds until the instance is destroyed; never free a possibly live key.
const hasTable = (name: string) => !!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name);
if (hasTable('ec_host_operations')) {
const payload = hasTable('ec_host_operation_payloads')
? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)' : 'o.payload';
const command = `CASE WHEN json_valid(${payload}) THEN ${payload} ELSE '{}' END`;
const launched = handle.prepare(`SELECT n.server,o.state,o.updated_at,
json_extract(${command},'$.env.PLUTO_SERVER_KEY') AS credential,
json_extract(${command},'$.game') AS game,
json_extract(${command},'$.env.PLUTAINER_GAME') AS game_id
FROM native_instances n JOIN ec_host_operations o ON o.instance_id=n.id AND o.host_id=n.host_id
WHERE n.retired_at IS NULL AND o.kind IN ('instance.create','instance.rebuild') AND o.state!='failed'
AND (o.generation=n.generation OR json_extract(${command},'$.nextGeneration')=n.generation)
ORDER BY o.rowid DESC`).all() as
{ server: string; state: string; updated_at: string; credential: unknown; game: unknown; game_id: unknown }[];
const recovered = new Map<string, string[]>();
const complete = new Set<string>();
for (const row of launched) {
if (complete.has(row.server) || typeof row.credential !== 'string' || !row.credential) continue;
const game = row.game === 't5' || row.game === 't6' ? row.game
: row.game_id === 't5sp' ? 't5' : row.game_id === 't6zm' ? 't6'
: /^(t5|t6)_/.exec(row.server)?.[1]
?? (serverGame.get(row.server) as { game: string } | undefined)?.game;
if (!game) throw new Error(`Cannot determine the game for launched server ${row.server}`);
const ids = recovered.get(row.server) ?? [];
const id = save(game, row.credential, row.server, row.updated_at);
if (!ids.includes(id)) ids.push(id);
recovered.set(row.server, ids);
if (row.state === 'completed') complete.add(row.server);
}
for (const [server, ids] of recovered) {
// If no completed launch receipt survives, its saved assignment
// remains a possible prior credential while the attempt resolves.
if (!complete.has(server)) for (const id of assignments.get(server) ?? []) if (!ids.includes(id)) ids.push(id);
assignments.set(server, ids);
}
}
for (const [server, ids] of assignments) ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0));
handle.exec('DROP TABLE legacy_server_keys');
}).immediate();
}