199 lines
14 KiB
TypeScript
199 lines
14 KiB
TypeScript
import Database from 'better-sqlite3';
|
|
import { createHash } from 'node:crypto';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import schema from '../../src/lib/server/db/schema.sql?raw';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
const server = 't5_zombie_theater_1';
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
|
|
vi.stubEnv('RCON_PASSWORD', '');
|
|
});
|
|
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); });
|
|
|
|
async function intent() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
|
|
ensureHostSchema(db());
|
|
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const id = beginDeployment(server, 'create');
|
|
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
|
|
db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)')
|
|
.run(id, JSON.stringify({ automatic: true, libraryId: 'library', image: 'plutainer:test' }));
|
|
db().prepare(`INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map)
|
|
VALUES(?,?,NULL,'zombie','theater','zombie_theater')`).run(id, server);
|
|
return id;
|
|
}
|
|
|
|
describe('native creation intents before placement', () => {
|
|
it('retains its identity and native history across restart without changing ownership', async () => {
|
|
const id = await intent();
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { getDeployment, interruptDeployments } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { nativeDeployment, unplacedNativeCreation } = await import('../../src/lib/server/backends/native.js');
|
|
const { nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
expect(nativeDeployment(id)).toBe(true);
|
|
expect(unplacedNativeCreation(server)).toBe(id);
|
|
expect(nextLobbyNumber('t5_zombie_theater')).toBe(2);
|
|
expect(requestLifecycle(server, 'stop')).toMatchObject({ ok: false, status: 409 });
|
|
closeDb();
|
|
interruptDeployments();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
expect(getDeployment(id)).toMatchObject({ backend: 'native', canCancelCreation: true, agentOperation: null,
|
|
deployment: { state: 'queued', server_exists: 0 } });
|
|
expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id)).toEqual({ port: null });
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
|
expect(db().prepare('SELECT deployment_id FROM native_provisioning').all()).toEqual([]);
|
|
});
|
|
|
|
it('cancels unplaced work locally once, retains keys, and frees its logical server name', async () => {
|
|
const id = await intent();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { saveServerKey, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { nativeDeployment, unplacedNativeCreation } = await import('../../src/lib/server/backends/native.js');
|
|
const { nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js');
|
|
saveServerKey(server, 'saved-key');
|
|
expect(canCancelCreation(id)).toBe(true);
|
|
const cleanup = cancelCreation(id)!;
|
|
expect(cleanup).toEqual(expect.any(String));
|
|
expect(cancelCreation(id)).toBe(cleanup);
|
|
expect(canCancelCreation(id)).toBe(false);
|
|
expect(nativeDeployment(cleanup)).toBe(true);
|
|
expect(getDeployment(id)).toMatchObject({ backend: 'native', cancellation: { state: 'done' }, deployment: { state: 'cancelled' } });
|
|
expect(getDeployment(cleanup)).toMatchObject({ backend: 'native', deployment: { state: 'done', server_deleted_at: expect.any(String) } });
|
|
expect(unplacedNativeCreation(server)).toBeNull();
|
|
expect(nextLobbyNumber('t5_zombie_theater')).toBe(1);
|
|
expect(getServerKey(server)).toBe('saved-key');
|
|
expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
});
|
|
|
|
it('will not cancel an unplaced historical intent against a replacement server', async () => {
|
|
const id = await intent();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("INSERT INTO servers(name) VALUES(?)").run(server);
|
|
const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(canCancelCreation(id)).toBe(false);
|
|
expect(cancelCreation(id)).toBeNull();
|
|
expect(db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toEqual({ name: server });
|
|
expect(db().prepare('SELECT cleanup_id FROM native_creation_cancellations').all()).toEqual([]);
|
|
});
|
|
|
|
it.each([false, true])('does not adopt a replacement instance during placement (previously bound: %s)', async previouslyBound => {
|
|
const id = await intent();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { createHost } = await import('../../src/lib/server/hosts/index.js');
|
|
const { captureRecipePlan, recipeStorage } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const hostId = createHost({ name: 'Replacement host' }).id;
|
|
const insert = db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at)
|
|
VALUES(?,?,?,'library','plutainer:test',28960,?,?)`);
|
|
if (previouslyBound) {
|
|
insert.run('original-instance', server, hostId, '2025-01-01', '2025-01-02');
|
|
db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(id, 'original-instance');
|
|
}
|
|
insert.run('replacement-instance', server, hostId, '2025-01-02', null);
|
|
const before = db().prepare('SELECT * FROM native_instances ORDER BY id').all();
|
|
const stored = recipeStorage(captureRecipePlan('t5', 'theater'));
|
|
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)')
|
|
.run(id, stored.recipe, stored.recipeSummary);
|
|
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
|
|
const placement = vi.spyOn(preparation, 'evaluateNativePlacement').mockRejectedValue(new Error('Unexpected placement'));
|
|
const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
|
|
await processNativeProvisioning(() => null);
|
|
expect(placement).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT state,error FROM server_deployments WHERE id=?').get(id))
|
|
.toEqual({ state: 'failed', error: 'Original instance was retired or replaced; no command was sent.' });
|
|
expect(db().prepare('SELECT * FROM native_instances ORDER BY id').all()).toEqual(before);
|
|
expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id)).toEqual({ port: null });
|
|
});
|
|
|
|
it('starts work on a ready host when an older pinned intent is waiting for an offline host', async () => {
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
|
const image = 'plutainer:test';
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const offline = hosts.createHost({ name: 'Offline host' });
|
|
const ready = hosts.createHost({ name: 'Ready host' });
|
|
const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
|
|
cpuCount: 16, cpuUsagePercent: 5, memoryTotal: 32 * 1024 ** 3, memoryAvailable: 30 * 1024 ** 3,
|
|
storageAvailable: 100_000_000_000, runtimeImages: [image] };
|
|
hosts.enrollHost(hosts.createEnrollment(ready.id).token, inventory);
|
|
const libraries = await import('../../src/lib/server/game-files/index.js');
|
|
const imported = libraries.createLibraryImport({ hostId: ready.id, game: 't5' });
|
|
const bytes = Buffer.from('verified base game');
|
|
const file = { path: 'zone/base.ff', size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') };
|
|
libraries.setImportManifest(ready.id, imported.id, [file]);
|
|
await libraries.uploadImportFile(ready.id, imported.id, file.path, new Response(bytes).body);
|
|
const library = await libraries.publishImport(ready.id, imported.id);
|
|
libraries.updateHostLibraryCaches(ready.id, [{ libraryVersionId: library.id, status: 'ready', manifestSha256: library.manifestSha256 }]);
|
|
hosts.heartbeat(ready.id, { ...inventory, libraries: [{ id: library.id, state: 'ready' }] });
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
|
|
const { queueNativeCreation, processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
|
|
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', libraryId: library.id, image, plutoKey: 'saved-key' };
|
|
const blocked = queueNativeCreation({ ...input, hostId: offline.id });
|
|
const available = queueNativeCreation({ ...input, hostId: ready.id });
|
|
expect(blocked.ok && available.ok).toBe(true);
|
|
if (!blocked.ok || !available.ok) throw new Error('Could not queue launch fixtures.');
|
|
await processNativeProvisioning(() => null);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(blocked.deploymentId)).toEqual({ state: 'queued' });
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(available.deploymentId)).toEqual({ state: 'running' });
|
|
expect(db().prepare("SELECT host_id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([{ host_id: ready.id }]);
|
|
expect(db().prepare('SELECT server,host_id FROM native_instances WHERE retired_at IS NULL').all())
|
|
.toEqual([{ server: available.name, host_id: ready.id }]);
|
|
expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(blocked.deploymentId)).toEqual({ port: null });
|
|
});
|
|
|
|
it('handles exact-match removal of an unplaced launch as local cancellation', async () => {
|
|
const id = await intent();
|
|
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const removal = destroyServer(server, 'blueprint-exact');
|
|
expect(removal).toMatchObject({ ok: true, deploymentId: expect.any(String) });
|
|
expect(getDeployment(id)?.deployment.state).toBe('cancelled');
|
|
expect(getDeployment(removal.deploymentId!)?.deployment.state).toBe('done');
|
|
expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]);
|
|
});
|
|
|
|
it('migrates existing port reservations without losing rows, references, indexes, or triggers', async () => {
|
|
const legacy = new Database(fixture.dbPath);
|
|
try {
|
|
legacy.exec(schema.replace(/CREATE TABLE IF NOT EXISTS server_creation_jobs \([\s\S]*?\n\);/,
|
|
table => table.replace(' port INTEGER,', ' port INTEGER NOT NULL,')));
|
|
legacy.exec(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
|
|
VALUES('existing','old-server','create','queued','Queued','2025-01-01','fixture');
|
|
INSERT INTO server_creation_jobs(rowid,deployment_id,server,port,game_mode,map_slug,expected_map)
|
|
VALUES(8,'existing','old-server',28960,'zombie','theater','zombie_theater');
|
|
CREATE INDEX fixture_creation_port ON server_creation_jobs(port);
|
|
CREATE TABLE fixture_creation_reference(id TEXT REFERENCES server_creation_jobs(deployment_id));
|
|
INSERT INTO fixture_creation_reference(id) VALUES('existing');
|
|
CREATE TABLE fixture_creation_audit(port INTEGER);
|
|
CREATE TRIGGER fixture_creation_update AFTER UPDATE OF port ON server_creation_jobs
|
|
BEGIN INSERT INTO fixture_creation_audit(port) VALUES(NEW.port); END;`);
|
|
} finally { legacy.close(); }
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
for (let open = 0; open < 2; open++) {
|
|
const handle = db();
|
|
expect(handle.prepare('SELECT rowid,deployment_id,port FROM server_creation_jobs').get())
|
|
.toEqual({ rowid: 8, deployment_id: 'existing', port: 28960 });
|
|
expect(handle.prepare('SELECT id FROM fixture_creation_reference').get()).toEqual({ id: 'existing' });
|
|
expect(handle.prepare("SELECT name FROM sqlite_master WHERE name IN ('fixture_creation_port','fixture_creation_update')").all()).toHaveLength(2);
|
|
expect(handle.pragma('foreign_key_check')).toEqual([]);
|
|
closeDb();
|
|
}
|
|
db().prepare('UPDATE server_creation_jobs SET port=NULL WHERE deployment_id=?').run('existing');
|
|
expect(db().prepare('SELECT port FROM fixture_creation_audit').all()).toEqual([{ port: null }]);
|
|
});
|
|
});
|