Queue server creation and verify health and selected map before completion
This commit is contained in:
+110
-24
@@ -1,4 +1,4 @@
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js';
|
||||
import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
@@ -6,7 +6,7 @@ import { config, destOfKind } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { getServerKey, saveServerKey } from './server-keys.js';
|
||||
import { getServer } from './servers.js';
|
||||
import { t5ZombieMapBySlug, type GameMode } from './map-aliases.js';
|
||||
import { t5ZombieMapBySlug, type GameMode, type T5MapEntry } from './map-aliases.js';
|
||||
import {
|
||||
komodoCreateDeployment,
|
||||
komodoDeploy,
|
||||
@@ -40,7 +40,7 @@ export type CreateServerFailure =
|
||||
| 'db'; // shouldn't happen, but wraps INSERT failures
|
||||
|
||||
export type CreateServerResult =
|
||||
| { ok: true; name: string; port: number }
|
||||
| { ok: true; name: string; port: number; deploymentId: string }
|
||||
| { ok: false; kind: CreateServerFailure; message: string };
|
||||
|
||||
export type DestroyServerResult =
|
||||
@@ -62,11 +62,14 @@ export interface CreateServerInput {
|
||||
*/
|
||||
function nextLobbyNumber(baseName: string): number | null {
|
||||
const rows = db()
|
||||
.prepare('SELECT name FROM servers WHERE name LIKE ?')
|
||||
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
|
||||
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
|
||||
.all(`${baseName}_%`) as Array<{ name: string }>;
|
||||
const taken = new Set<number>();
|
||||
const prefix = `${baseName}_`;
|
||||
for (const r of rows) {
|
||||
if (!r.name.startsWith(prefix)) continue;
|
||||
const tail = r.name.slice(prefix.length);
|
||||
if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10));
|
||||
}
|
||||
@@ -83,7 +86,9 @@ function nextLobbyNumber(baseName: string): number | null {
|
||||
*/
|
||||
function nextAvailablePort(): number | null {
|
||||
const rows = db()
|
||||
.prepare('SELECT game_port FROM servers WHERE game_port IS NOT NULL')
|
||||
.prepare(`SELECT game_port FROM servers WHERE game_port IS NOT NULL UNION
|
||||
SELECT j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
|
||||
.all() as Array<{ game_port: number }>;
|
||||
const taken = new Set(rows.map((r) => r.game_port));
|
||||
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
|
||||
@@ -139,25 +144,32 @@ function buildSpec(args: {
|
||||
}
|
||||
|
||||
let creating = false;
|
||||
export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
|
||||
if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' };
|
||||
creating = true;
|
||||
let operation: string | undefined;
|
||||
try {
|
||||
const result = await createServerUnlocked(input, (name) => {
|
||||
operation = beginDeployment(name, 'create', input.createdBy);
|
||||
return operation;
|
||||
});
|
||||
if (operation) finishDeployment(operation, result);
|
||||
return result;
|
||||
} catch (err) {
|
||||
if (operation) finishDeployment(operation, { ok: false, message: (err as Error).message });
|
||||
throw err;
|
||||
}
|
||||
finally { creating = false; }
|
||||
interface PreparedCreation {
|
||||
ok: true; name: string; port: number; plutoKey: string; mapEntry: T5MapEntry;
|
||||
}
|
||||
|
||||
async function createServerUnlocked(input: CreateServerInput, begin: (name: string) => string): Promise<CreateServerResult> {
|
||||
export function queueCreateServer(input: CreateServerInput): CreateServerResult {
|
||||
return db().transaction((): CreateServerResult => {
|
||||
const prepared = prepareCreation(input);
|
||||
if (!prepared.ok) return prepared;
|
||||
const id = beginDeployment(prepared.name, 'create', input.createdBy);
|
||||
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
|
||||
deploymentStage(id, 'Queued for server creation');
|
||||
return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
|
||||
})();
|
||||
}
|
||||
|
||||
// Await provisioning for internal callers; completion still requires worker health verification.
|
||||
export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
|
||||
if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' };
|
||||
const queued = queueCreateServer(input);
|
||||
if (!queued.ok) return queued;
|
||||
return await provisionCreation(queued.deploymentId);
|
||||
}
|
||||
|
||||
function prepareCreation(input: CreateServerInput): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
|
||||
if (input.gameMode !== 'zombie') {
|
||||
return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` };
|
||||
}
|
||||
@@ -198,7 +210,11 @@ async function createServerUnlocked(input: CreateServerInput, begin: (name: stri
|
||||
try { saveServerKey(name, plutoKey); }
|
||||
catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; }
|
||||
|
||||
const operation = begin(name);
|
||||
return { ok: true, name, port, plutoKey, mapEntry };
|
||||
}
|
||||
|
||||
async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise<CreateServerResult> {
|
||||
const { name, port, plutoKey, mapEntry } = prepared;
|
||||
deploymentStage(operation, 'Preparing server files');
|
||||
const hostDir = `${config.serversRoot}/${name}`;
|
||||
const initialConfig = getPromotedServerConfig('t5', mapEntry.canonical);
|
||||
@@ -304,7 +320,7 @@ async function createServerUnlocked(input: CreateServerInput, begin: (name: stri
|
||||
// now instead of on the next 15s tick.
|
||||
void kick().catch((err) => console.error('[createServer] post-create kick failed:', err));
|
||||
|
||||
return { ok: true, name, port };
|
||||
return { ok: true, name, port, deploymentId: operation };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -453,5 +469,75 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
|
||||
};
|
||||
}
|
||||
|
||||
const unfinished = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'awaiting-health'").all(name) as { id: string }[];
|
||||
for (const { id } of unfinished) finishDeployment(id, { ok: false, message: 'Server destroyed before creation health checks completed' });
|
||||
return { ok: true, warnings };
|
||||
}␍interface CreationJob {
|
||||
deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
|
||||
}
|
||||
|
||||
async function provisionCreation(id: string): Promise<CreateServerResult> {
|
||||
const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob;
|
||||
creating = true;
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id);
|
||||
try {
|
||||
const mapEntry = t5ZombieMapBySlug(job.map_slug);
|
||||
const plutoKey = getServerKey(job.server);
|
||||
if (!mapEntry || !plutoKey) {
|
||||
const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' };
|
||||
finishDeployment(id, result);
|
||||
return result;
|
||||
}
|
||||
const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
|
||||
{ ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
|
||||
if (!result.ok) finishDeployment(id, result);
|
||||
else {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for Komodo health');
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
finishDeployment(id, { ok: false, message: (err as Error).message });
|
||||
throw err;
|
||||
} finally { creating = false; }
|
||||
}
|
||||
|
||||
let creationTick: Promise<void> | null = null;
|
||||
export function processCreations(): Promise<void> {
|
||||
if (creationTick) return creationTick;
|
||||
creationTick = runCreations().finally(() => { creationTick = null; });
|
||||
return creationTick;
|
||||
}
|
||||
|
||||
async function runCreations(): Promise<void> {
|
||||
if (creating || destroying.size > 0) return;
|
||||
const queued = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'queued' ORDER BY started_at").all() as { id: string }[];
|
||||
for (const { id } of queued) {
|
||||
if (creating || destroying.size > 0) break;
|
||||
await provisionCreation(id).catch(err => console.error('[creation] provisioning failed:', err));
|
||||
}
|
||||
const checking = db().prepare("SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state = 'awaiting-health'").all() as CreationJob[];
|
||||
for (const job of checking) {
|
||||
const id = job.deployment_id;
|
||||
if (!getServer(job.server)) {
|
||||
finishDeployment(id, { ok: false, message: 'Server was removed before creation health checks completed' });
|
||||
continue;
|
||||
}
|
||||
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
|
||||
try {
|
||||
const komodo = await komodoInspectHealthy(job.server);
|
||||
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
|
||||
const rcon = await rconStatusResult(job.server);
|
||||
if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; }
|
||||
const actualMap = rcon.status.map;
|
||||
if (actualMap !== job.expected_map) {
|
||||
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map'));
|
||||
continue;
|
||||
}
|
||||
deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap);
|
||||
finishDeployment(id, { ok: true });
|
||||
} catch (err) {
|
||||
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user