4573 lines
216 KiB
TypeScript
4573 lines
216 KiB
TypeScript
import { putFileBlob } from '../../src/lib/server/db/file-blobs.js';
|
|
import { createHash } from 'node:crypto';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
const remote = vi.hoisted(() => ({ state: 'running', players: 0, map: 'zombie_theater' }));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconStatusResult: vi.fn(async () => ({
|
|
ok: true,
|
|
endpoint: { host: 'agent', port: 28960 },
|
|
status: { players: remote.players, map: remote.map, gametype: 'zom', roster: [] }
|
|
})),
|
|
rconGetPlayerCount: vi.fn(async () => remote.players),
|
|
rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
|
|
}));
|
|
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({
|
|
checkRelease: vi.fn(async () => {}),
|
|
freshRelease: () => 5354,
|
|
probeVersion: vi.fn(async () => 5354)
|
|
}));
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({
|
|
kick: vi.fn(async () => {}),
|
|
kickProvisioning: vi.fn(async () => {})
|
|
}));
|
|
vi.mock('../../src/lib/server/rebalancing/admission.js', () => ({
|
|
rebalanceAdmission: vi.fn(() => null),
|
|
requestRebalanceStopForOperation: vi.fn()
|
|
}));
|
|
|
|
let queuedKeys = 0;
|
|
let fixture: Fixture;
|
|
let hostId: string;
|
|
let libraryId: string;
|
|
const image = 'example.test/plutainer:test';
|
|
const inventory = () => ({
|
|
cpuCount: 16,
|
|
cpuUsagePercent: 10,
|
|
memoryTotal: 64 * 1024 ** 3,
|
|
memoryAvailable: 60 * 1024 ** 3,
|
|
protocolVersion: 1,
|
|
capabilities: ['instance.unhealthy-recovery', 'instance.rebuild-files'],
|
|
dockerRunning: true,
|
|
directoriesWritable: true,
|
|
storageAvailable: 100_000_000_000,
|
|
preparedRuntimeImages: [image],
|
|
libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : []
|
|
});
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.clearAllMocks();
|
|
remote.state = 'running';
|
|
remote.players = 0;
|
|
remote.map = 'zombie_theater';
|
|
libraryId = '';
|
|
queuedKeys = 0;
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
|
vi.stubEnv('PLUTAINER_IMAGE', image);
|
|
vi.stubEnv('RCON_PASSWORD', 'test-password');
|
|
|
|
const { createHost, createEnrollment, enrollHost } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
hostId = createHost({ name: 'Native Debian', portMin: 28960, portMax: 28970 }).id;
|
|
enrollHost(createEnrollment(hostId).token, inventory());
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => {
|
|
if (kind !== 'instance.inspect') throw new Error(`Unexpected operation: ${kind}`);
|
|
return { healthy: remote.state === 'running', state: remote.state, exists: true } as never;
|
|
});
|
|
});
|
|
afterEach(async () => {
|
|
vi.useRealTimers();
|
|
await fixture.cleanup();
|
|
vi.unstubAllEnvs();
|
|
});
|
|
|
|
async function prepareLibrary(game: 't5' | 't6' = 't5') {
|
|
const libs = await import('../../src/lib/server/game-files/index.js');
|
|
const version = libs.createLibraryImport({ hostId, game });
|
|
const bytes = Buffer.from('verified base game');
|
|
const file = {
|
|
path: 'zone/base.ff',
|
|
size: bytes.length,
|
|
sha256: createHash('sha256').update(bytes).digest('hex')
|
|
};
|
|
libs.setImportManifest(hostId, version.id, [file]);
|
|
await libs.uploadImportFile(hostId, version.id, file.path, new Response(bytes).body);
|
|
const ready = await libs.publishImport(hostId, version.id);
|
|
libraryId = ready.id;
|
|
libs.updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }
|
|
]);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, inventory());
|
|
return ready;
|
|
}
|
|
async function queue(game: 't5' | 't6' = 't5', placement: 'pinned' | 'automatic' = 'pinned') {
|
|
await prepareLibrary(game);
|
|
remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit';
|
|
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const result = queueCreateServer({
|
|
game,
|
|
gameMode: 'zombie',
|
|
mapSlug: game === 't5' ? 'theater' : 'town',
|
|
hostId: placement === 'pinned' ? hostId : undefined,
|
|
libraryId,
|
|
image,
|
|
plutoKey: ++queuedKeys === 1 ? 'saved-key' : `saved-key-${queuedKeys}`
|
|
});
|
|
if (!result.ok) throw new Error(result.message);
|
|
return result;
|
|
}
|
|
async function operationFor(deploymentId: string) {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const row = db()
|
|
.prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?')
|
|
.get(deploymentId) as { operation_id: string | null };
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
return row?.operation_id ? getHostOperation(row.operation_id)! : null;
|
|
}
|
|
async function advance() {
|
|
const { processNativeProvisioning } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
await processNativeProvisioning(() => null);
|
|
}
|
|
async function acknowledge(deploymentId: string) {
|
|
const operation = await operationFor(deploymentId);
|
|
if (!operation) throw new Error('Missing durable native operation');
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { state: remote.state, healthy: remote.state === 'running' }
|
|
});
|
|
await advance();
|
|
return operation;
|
|
}
|
|
async function detail(id: string) {
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
return getDeployment(id)!.deployment;
|
|
}
|
|
|
|
async function observations(server: string) {
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
const { serverObservation, observationReader } =
|
|
await import('../../src/lib/server/fleet/status.js');
|
|
const { MonitoringSnapshot } =
|
|
await import('../../src/lib/server/fleet/monitoring-snapshot.js');
|
|
const row = getServer(server)!;
|
|
return [
|
|
serverObservation(row),
|
|
observationReader()(row),
|
|
new MonitoringSnapshot().observe(row)
|
|
];
|
|
}
|
|
|
|
async function moveTarget() {
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const libraries = await import('../../src/lib/server/game-files/index.js');
|
|
const target = hosts.createHost({
|
|
name: 'Rebalance destination',
|
|
portMin: 29000,
|
|
portMax: 29005
|
|
});
|
|
hosts.enrollHost(hosts.createEnrollment(target.id).token, inventory());
|
|
libraries.updateHostLibraryCaches(target.id, [
|
|
{
|
|
libraryVersionId: libraryId,
|
|
status: 'ready',
|
|
manifestSha256: libraries.getLibraryVersion(libraryId)!.manifestSha256
|
|
}
|
|
]);
|
|
return target;
|
|
}
|
|
|
|
describe('rebalance instance migrations', () => {
|
|
it('preserves deployed configuration, patched file versions, identity and credentials on an active source', async () => {
|
|
const { config } = await import('../../src/lib/server/config.js');
|
|
Object.assign(config.rcon, { password: '' });
|
|
const { createScript, uploadNewVersion } =
|
|
await import('../../src/lib/server/fleet/scripts.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const baseline = createScript({
|
|
name: 'Deployed baseline',
|
|
kind: 'baseline',
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content: Buffer.from('set sv_voice "1"\n')
|
|
});
|
|
const script = createScript({
|
|
releaseVersion: '1.0.0',
|
|
name: 'Deployed script',
|
|
kind: 'script',
|
|
targetRelPath: 'preserved.gsc',
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content: Buffer.from('original deployed script')
|
|
});
|
|
for (const artifact of [baseline, script])
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(artifact.version.id, artifact.script.id);
|
|
const job = await queue();
|
|
await advance();
|
|
const launch = (await operationFor(job.deploymentId))!.payload as {
|
|
rconPassword: string;
|
|
files: { path: string; sha256: string }[];
|
|
};
|
|
await acknowledge(job.deploymentId);
|
|
const patched = uploadNewVersion(
|
|
script.script.id,
|
|
Buffer.from('confirmed patch after initial launch'),
|
|
undefined,
|
|
'1.0.1'
|
|
);
|
|
const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
recordScript(
|
|
job.name,
|
|
'preserved.gsc',
|
|
patched.id,
|
|
patched.sha256,
|
|
patched.size,
|
|
'campaign'
|
|
);
|
|
const newest = uploadNewVersion(
|
|
script.script.id,
|
|
Buffer.from('new promotion must not be installed'),
|
|
undefined,
|
|
'1.0.2'
|
|
);
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(newest.id, script.script.id);
|
|
const newBaseline = uploadNewVersion(baseline.script.id, Buffer.from('set sv_voice "0"\n'));
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(newBaseline.id, baseline.script.id);
|
|
Object.assign(config.rcon, { password: 'new-controller-password' });
|
|
const target = await moveTarget();
|
|
const { captureMoveRecipe, requestInstanceMove, hostMigration } =
|
|
await import('../../src/lib/server/provisioning/host-drain.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const original = nativeInstance(job.name)!;
|
|
const recipe = captureMoveRecipe(job.name);
|
|
expect(recipe.files.every((file) => file.content === undefined)).toBe(true);
|
|
const id = requestInstanceMove(job.name, target.id, {
|
|
triggeredBy: 'rebalance:test',
|
|
recipe
|
|
});
|
|
await advance();
|
|
const removal = hostMigration(id)!.remove_operation_id!;
|
|
expect(removal).toBeTruthy();
|
|
const { cancelProvisioningJob, canCancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
expect(canCancelProvisioningJob(id)).toBe(false);
|
|
expect(cancelProvisioningJob(id)).toBe(false);
|
|
const { getDeployment, listDeployments } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
expect(getDeployment(id)?.canCancel).toBe(false);
|
|
expect(listDeployments().find((item) => item.id === id)?.canCancel).toBe(0);
|
|
const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js');
|
|
const response = POST({ params: { id } });
|
|
expect(response.status).toBe(409);
|
|
expect(await response.json()).toMatchObject({
|
|
error: expect.stringContaining('must finish recreating')
|
|
});
|
|
const { reportHostOperation, getHost } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, removal, { state: 'completed' });
|
|
await advance();
|
|
await advance();
|
|
const creation = (await operationFor(id))!;
|
|
expect(creation).toMatchObject({
|
|
hostId: target.id,
|
|
kind: 'instance.create',
|
|
generation: 2,
|
|
payload: {
|
|
name: job.name,
|
|
libraryId,
|
|
image,
|
|
rconPassword: launch.rconPassword,
|
|
env: { PLUTO_SERVER_KEY: 'saved-key' }
|
|
}
|
|
});
|
|
const payload = creation.payload as { files: { path: string; sha256: string }[] };
|
|
expect(payload.files).toContainEqual(
|
|
expect.objectContaining({
|
|
path: expect.stringContaining('preserved.gsc'),
|
|
sha256: patched.sha256
|
|
})
|
|
);
|
|
expect(payload.files.find((file) => file.path === 'configs/dedicated_sp.cfg')?.sha256).toBe(
|
|
launch.files.find((file) => file.path === 'configs/dedicated_sp.cfg')?.sha256
|
|
);
|
|
reportHostOperation(target.id, creation.id, {
|
|
state: 'completed',
|
|
result: { state: 'running', healthy: true }
|
|
});
|
|
await advance();
|
|
expect((await detail(id)).state).toBe('done');
|
|
expect(nativeInstance(job.name)).toMatchObject({
|
|
id: original.id,
|
|
host_id: target.id,
|
|
generation: 2
|
|
});
|
|
expect(getHost(hostId)?.mode).toBe('active');
|
|
expect(JSON.stringify(getDeployment(id)?.recipe)).not.toContain(launch.rconPassword);
|
|
});
|
|
|
|
it('rechecks occupancy, target capacity and run admission before removing the source', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const target = await moveTarget();
|
|
const { requestInstanceMove, hostMigration } =
|
|
await import('../../src/lib/server/provisioning/host-drain.js');
|
|
expect(() =>
|
|
requestInstanceMove(job.name, hostId, { triggeredBy: 'rebalance:test' })
|
|
).toThrow('different destination');
|
|
const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' });
|
|
remote.players = 2;
|
|
await advance();
|
|
expect(hostMigration(id)?.remove_operation_id).toBeNull();
|
|
remote.players = 0;
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(target.id, { ...inventory(), memoryAvailable: 1 });
|
|
await advance();
|
|
expect(hostMigration(id)?.remove_operation_id).toBeNull();
|
|
heartbeat(target.id, inventory());
|
|
const { rebalanceAdmission } =
|
|
await import('../../src/lib/server/rebalancing/admission.js');
|
|
vi.mocked(rebalanceAdmission).mockReturnValueOnce('Run stopped during occupancy check.');
|
|
await advance();
|
|
expect(hostMigration(id)?.remove_operation_id).toBeNull();
|
|
await advance();
|
|
expect(hostMigration(id)?.remove_operation_id).toEqual(expect.any(String));
|
|
});
|
|
|
|
it('withdraws only uncommitted moves and preserves their server keys', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const target = await moveTarget();
|
|
const { requestInstanceMove } =
|
|
await import('../../src/lib/server/provisioning/host-drain.js');
|
|
const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' });
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
expect(cancelProvisioningJob(id)).toBe(true);
|
|
const { requestRebalanceStopForOperation } =
|
|
await import('../../src/lib/server/rebalancing/admission.js');
|
|
expect(requestRebalanceStopForOperation).toHaveBeenCalledWith(id);
|
|
await advance();
|
|
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
expect(getServerKey(job.name)).toBe('saved-key');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()
|
|
).toEqual([]);
|
|
});
|
|
|
|
it('refuses to move when recorded deployed bytes cannot be restored', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { captureMoveRecipe } =
|
|
await import('../../src/lib/server/provisioning/host-drain.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare('UPDATE server_scripts SET sha256=? WHERE server=?')
|
|
.run('0'.repeat(64), job.name);
|
|
expect(() => captureMoveRecipe(job.name)).toThrow('deployed bytes are unavailable');
|
|
});
|
|
});
|
|
|
|
describe('runtime configuration at dispatch', () => {
|
|
it.each([
|
|
['t5', 'shared'],
|
|
['t5', 'generated'],
|
|
['t6', 'shared'],
|
|
['t6', 'generated']
|
|
] as const)(
|
|
'dispatches %s with a promoted baseline and %s RCON password',
|
|
async (game, source) => {
|
|
const { config, configFileForGame } = await import('../../src/lib/server/config.js');
|
|
if (source === 'generated') Object.assign(config.rcon, { password: '' });
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { script, version } = createScript({
|
|
name: 'Launch baseline',
|
|
kind: 'baseline',
|
|
scopeType: 'game',
|
|
scopeValue: game,
|
|
content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n')
|
|
});
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(version.id, script.id);
|
|
|
|
const job = await queue(game);
|
|
await advance();
|
|
const operation = await operationFor(job.deploymentId);
|
|
expect(operation).toMatchObject({ kind: 'instance.create', state: 'pending' });
|
|
const payload = operation!.payload as {
|
|
rconPassword: string;
|
|
files: { path: string; contentBase64: string; sha256: string }[];
|
|
};
|
|
expect(payload.rconPassword).toEqual(
|
|
source === 'shared' ? 'test-password' : expect.stringMatching(/^[a-f0-9]{64}$/)
|
|
);
|
|
const configFile = configFileForGame(game);
|
|
const file = payload.files.find((file) => file.path === `configs/${configFile}`)!;
|
|
const content = Buffer.from(file.contentBase64, 'base64');
|
|
expect(content.toString()).toContain(`set rcon_password "${payload.rconPassword}"`);
|
|
expect(content.toString()).not.toContain('set rcon_password "password"');
|
|
const hash = createHash('sha256').update(content).digest('hex');
|
|
expect(file.sha256).toBe(hash);
|
|
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
expect(getServerScript(job.name, configFile)).toMatchObject({
|
|
sha256: hash,
|
|
size: content.length,
|
|
source: 'native-create'
|
|
});
|
|
const { recipeForDeployment } =
|
|
await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
expect(recipeForDeployment(job.deploymentId)!.recipe.files).toContainEqual(
|
|
expect.objectContaining({
|
|
path: configFile,
|
|
sha256: hash,
|
|
size: content.length,
|
|
content: content.toString('base64')
|
|
})
|
|
);
|
|
}
|
|
);
|
|
});
|
|
|
|
describe('provisioning health handoff', () => {
|
|
it.each(['create', 'start', 'restart', 'rebuild'] as const)(
|
|
'%s publishes verified game health before completion becomes visible',
|
|
async (action) => {
|
|
const job = await queue();
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
let id = job.deploymentId;
|
|
if (action !== 'create') {
|
|
await acknowledge(id);
|
|
db()
|
|
.prepare('UPDATE servers SET container_state=? WHERE name=?')
|
|
.run(action === 'start' ? 'stopped' : 'running', job.name);
|
|
const { requestLifecycle } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
const request = requestLifecycle(job.name, action);
|
|
if (!request.ok) throw new Error(request.error);
|
|
id = request.deploymentId;
|
|
await advance();
|
|
}
|
|
const old = new Date(Date.now() - 120_000).toISOString();
|
|
db()
|
|
.prepare(
|
|
`UPDATE servers SET last_seen_at=?,last_checked_at=?,last_players=0,last_roster='[]',
|
|
last_error='timeout:old process',rcon_failures=2,active_since=NULL,inactive_since=? WHERE name=?`
|
|
)
|
|
.run(old, old, old, job.name);
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(Date.now() + 1000);
|
|
if (action === 'rebuild') {
|
|
// A new generation now establishes its own durable evidence boundary.
|
|
// Observe game health strictly after that transition.
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
vi.mocked(nativeRequest).mockImplementationOnce(async () => {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
return { healthy: true, state: 'running', exists: true } as never;
|
|
});
|
|
}
|
|
const observedAt = new Date(Date.now() + (action === 'rebuild' ? 1 : 0)).toISOString();
|
|
const roster = [
|
|
{ num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }
|
|
];
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconStatusResult).mockResolvedValueOnce({
|
|
ok: true,
|
|
endpoint: { host: 'agent', port: 28960 },
|
|
status: { players: 1, map: remote.map, gametype: 'zom', roster }
|
|
});
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
|
|
const visible: string[] = [];
|
|
const unsubscribe = subscribe((event) => {
|
|
if (
|
|
event.type === 'fleet-invalidated' &&
|
|
getDeployment(id)?.deployment.state === 'done'
|
|
) {
|
|
visible.push(serverObservation(getServer(job.name)!).health);
|
|
}
|
|
});
|
|
try {
|
|
await acknowledge(id);
|
|
} finally {
|
|
unsubscribe();
|
|
}
|
|
expect((await detail(id)).state).toBe('done');
|
|
expect(visible).toEqual(['online']);
|
|
expect(getServer(job.name)).toMatchObject({
|
|
last_checked_at: observedAt,
|
|
last_seen_at: observedAt,
|
|
last_players: 1,
|
|
last_roster: JSON.stringify(roster),
|
|
last_reply_snippet: null,
|
|
last_error: null,
|
|
rcon_failures: 0,
|
|
active_since: observedAt,
|
|
inactive_since: null
|
|
});
|
|
for (const status of await observations(job.name))
|
|
expect(status).toMatchObject({ health: 'online', last_players: 1, roster });
|
|
}
|
|
);
|
|
|
|
it('completes directly from persisted telemetry without inventing an RCON observation', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { state: 'running', healthy: true }
|
|
});
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(Date.now() + 1000);
|
|
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
|
const { gameEvent } = await import('../helpers/telemetry.js');
|
|
const receivedAt = new Date().toISOString();
|
|
saveTelemetry({ ...gameEvent('heartbeat', 0, 0, job.name), receivedAt });
|
|
await advance();
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
expect(getServer(job.name)).toMatchObject({
|
|
last_seen_at: null,
|
|
inactive_since: receivedAt
|
|
});
|
|
for (const status of await observations(job.name))
|
|
expect(status).toMatchObject({
|
|
health: 'online',
|
|
status_source: 'telemetry',
|
|
last_seen_at: receivedAt
|
|
});
|
|
});
|
|
|
|
it('rechecks expired verification evidence instead of renewing its timestamps', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(Date.now() + 1000);
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
vi.mocked(probeVersion).mockImplementationOnce(async () => {
|
|
vi.setSystemTime(Date.now() + 91_000);
|
|
heartbeat(hostId, inventory());
|
|
return 5354;
|
|
});
|
|
await acknowledge(job.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('awaiting-health');
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
expect(getServer(job.name)?.last_seen_at).toBeNull();
|
|
for (const status of await observations(job.name))
|
|
expect(status.health).toBe('provisioning');
|
|
await advance();
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
expect(getServer(job.name)?.last_seen_at).toBe(new Date().toISOString());
|
|
});
|
|
|
|
it.each(['instance', 'lifecycle'] as const)(
|
|
'discards verification when the %s changes during later checks',
|
|
async (change) => {
|
|
const job = await queue();
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
vi.mocked(probeVersion).mockImplementationOnce(async () => {
|
|
if (change === 'instance')
|
|
db()
|
|
.prepare(
|
|
'UPDATE native_instances SET generation=generation+1 WHERE server=?'
|
|
)
|
|
.run(job.name);
|
|
else
|
|
db()
|
|
.prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?')
|
|
.run(new Date(Date.now() + 1000).toISOString(), job.deploymentId);
|
|
return 5354;
|
|
});
|
|
await acknowledge(job.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('awaiting-health');
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
expect(getServer(job.name)).toMatchObject({
|
|
last_seen_at: null,
|
|
container_checked_at: null
|
|
});
|
|
}
|
|
);
|
|
|
|
it.each(['online', 'offline'] as const)(
|
|
'preserves newer %s monitoring evidence received during version verification',
|
|
async (health) => {
|
|
const job = await queue();
|
|
await advance();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(Date.now() + 1000);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
let newer: string;
|
|
vi.mocked(probeVersion).mockImplementationOnce(async () => {
|
|
vi.setSystemTime(Date.now() + 1000);
|
|
newer = new Date().toISOString();
|
|
db()
|
|
.prepare(
|
|
`UPDATE servers SET container_state=?,container_checked_at=?,last_seen_at=?,last_checked_at=?,
|
|
last_players=2,last_roster='[]',last_error=NULL WHERE name=?`
|
|
)
|
|
.run(
|
|
health === 'online' ? 'running' : 'stopped',
|
|
newer,
|
|
newer,
|
|
newer,
|
|
job.name
|
|
);
|
|
return 5354;
|
|
});
|
|
await acknowledge(job.deploymentId);
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
expect(getServer(job.name)).toMatchObject({
|
|
last_seen_at: newer!,
|
|
container_checked_at: newer!,
|
|
last_players: 2
|
|
});
|
|
expect((await detail(job.deploymentId)).state).toBe(
|
|
health === 'online' ? 'done' : 'awaiting-health'
|
|
);
|
|
if (health === 'online')
|
|
for (const status of await observations(job.name))
|
|
expect(status).toMatchObject({ health: 'online', last_players: 2 });
|
|
}
|
|
);
|
|
|
|
it('keeps a slow restart Starting until verification succeeds and surfaces terminal failures', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const request = requestLifecycle(job.name, 'restart');
|
|
if (!request.ok) throw new Error(request.error);
|
|
await advance();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(Date.now() + 20_000);
|
|
for (const status of await observations(job.name)) expect(status.health).toBe('starting');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
for (const state of ['failed', 'cancelled', 'interrupted']) {
|
|
db()
|
|
.prepare('UPDATE server_deployments SET state=? WHERE id=?')
|
|
.run(state, request.deploymentId);
|
|
for (const status of await observations(job.name))
|
|
expect(status.health).toBe('offline');
|
|
}
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state='running' WHERE id=?")
|
|
.run(request.deploymentId);
|
|
await acknowledge(request.deploymentId);
|
|
expect((await detail(request.deploymentId)).state).toBe('done');
|
|
for (const status of await observations(job.name)) expect(status.health).toBe('online');
|
|
});
|
|
});
|
|
|
|
describe('native instance provisioning', () => {
|
|
it('recognizes an unplaced native intent and resumes it once after a controller restart', async () => {
|
|
const job = await queue();
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { nativeDeployment, nativeInstance } =
|
|
await import('../../src/lib/server/backends/native.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
expect(job.port).toBeNull();
|
|
expect(nativeDeployment(job.deploymentId)).toBe(true);
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
expect(getDeployment(job.deploymentId)).toMatchObject({
|
|
backend: 'native',
|
|
canCancelCreation: true
|
|
});
|
|
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([
|
|
{ deployment_id: job.deploymentId }
|
|
]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
|
|
closeDb();
|
|
vi.resetModules();
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
expect(backend.nativeDeployment(job.deploymentId)).toBe(true);
|
|
await advance();
|
|
const original = await operationFor(job.deploymentId);
|
|
expect(original).toMatchObject({
|
|
kind: 'instance.create',
|
|
payload: { name: job.name, port: 28960 }
|
|
});
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))?.id).toBe(original!.id);
|
|
const restartedDb = (await import('../../src/lib/server/db/index.js')).db();
|
|
expect(
|
|
restartedDb
|
|
.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'")
|
|
.all()
|
|
).toHaveLength(1);
|
|
expect(
|
|
restartedDb.prepare('SELECT protocol FROM ec_host_ports ORDER BY protocol').all()
|
|
).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]);
|
|
});
|
|
|
|
it('still rejects missing keys and stale lobby identities before creating an intent', async () => {
|
|
await prepareLibrary();
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const input = {
|
|
game: 't5' as const,
|
|
gameMode: 'zombie' as const,
|
|
mapSlug: 'theater',
|
|
hostId,
|
|
libraryId,
|
|
image
|
|
};
|
|
expect(queueNativeCreation(input)).toMatchObject({
|
|
ok: false,
|
|
message: expect.stringContaining('server key')
|
|
});
|
|
expect(
|
|
queueNativeCreation({
|
|
...input,
|
|
plutoKey: 'saved-key',
|
|
expectedName: 't5_zombie_theater_2'
|
|
})
|
|
).toMatchObject({ ok: false });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([]);
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
});
|
|
|
|
it('accepts creation cleanup through the provisioning API and rejects an already completed creation', async () => {
|
|
const job = await queue();
|
|
const { POST } =
|
|
await import('../../src/routes/api/provisioning/[id]/cancel-creation/+server.js');
|
|
const response = POST({ params: { id: job.deploymentId } });
|
|
expect(response.status).toBe(200);
|
|
expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) });
|
|
const next = await queue();
|
|
await advance();
|
|
await acknowledge(next.deploymentId);
|
|
expect(POST({ params: { id: next.deploymentId } }).status).toBe(409);
|
|
expect(POST({ params: { id: job.deploymentId } }).status).toBe(200);
|
|
});
|
|
it('cancels an unsubmitted creation locally and returns the same cleanup on repeated requests', async () => {
|
|
const job = await queue();
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(getDeployment(job.deploymentId)!.canCancelCreation).toBe(true);
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
expect(cancelCreation(job.deploymentId)).toBe(cleanup);
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect((await detail(cleanup)).state).toBe('done');
|
|
expect(getDeployment(job.deploymentId)!.cancellation?.state).toBe('done');
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
await advance();
|
|
expect(await operationFor(job.deploymentId)).toBeNull();
|
|
expect(await operationFor(cleanup)).toBeNull();
|
|
});
|
|
|
|
it('cancels a submitted creation without occupancy checks, suppresses replay, and releases resources only after cleanup', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
remote.players = 4;
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const { heartbeat, pendingHostOperations, reportHostOperation } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
|
|
// A previously delivered command might still be executing when cancellation arrives.
|
|
expect(pendingHostOperations(hostId).map((op) => op.id)).toContain(original.id);
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
const removal = (await operationFor(cleanup))!;
|
|
expect(removal.kind).toBe('instance.cancel-create');
|
|
expect(removal.payload).toEqual({ createOperationId: original.id });
|
|
expect(cancelCreation(job.deploymentId)).toBe(cleanup);
|
|
expect(pendingHostOperations(hostId).map((op) => op.id)).toContain(removal.id);
|
|
expect(pendingHostOperations(hostId).map((op) => op.id)).not.toContain(original.id);
|
|
await advance();
|
|
expect(nativeRequest).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
|
|
reportHostOperation(hostId, original.id, { state: 'completed', result: { healthy: true } });
|
|
await advance();
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect(nativeRequest).not.toHaveBeenCalled();
|
|
reportHostOperation(hostId, removal.id, { state: 'completed', result: { exists: false } });
|
|
await advance();
|
|
expect((await detail(cleanup)).state).toBe('done');
|
|
expect((await detail(job.deploymentId)).server_deleted_at).toBeTruthy();
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
|
|
expect(
|
|
db()
|
|
.prepare('SELECT retired_at FROM native_instances WHERE id=?')
|
|
.get(original.instanceId)
|
|
).toMatchObject({ retired_at: expect.any(String) });
|
|
});
|
|
|
|
it('holds cancellation for an old or offline host without delivering an unsupported command or losing reservations', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const { heartbeat, pendingHostOperations } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
expect(
|
|
pendingHostOperations(hostId).filter((op) => op.kind.startsWith('instance.'))
|
|
).toEqual([]);
|
|
await advance();
|
|
expect((await detail(cleanup)).stage).toContain('Update the host agent');
|
|
expect((await detail(cleanup)).stage).toContain('Automatic updates are not enabled');
|
|
heartbeat(hostId, {
|
|
...inventory(),
|
|
capabilities: ['agent.update'],
|
|
agentUpdate: {
|
|
enabled: true,
|
|
state: 'deferred',
|
|
currentVersion: '1.2.2',
|
|
targetVersion: '1.3.0'
|
|
}
|
|
});
|
|
await advance();
|
|
expect((await detail(cleanup)).stage).toContain('New command delivery is paused');
|
|
expect((await detail(cleanup)).stage).toContain('automatic updater can run');
|
|
db()
|
|
.prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?')
|
|
.run('2000-01-01T00:00:00.000Z', hostId);
|
|
await advance();
|
|
expect((await detail(cleanup)).stage).toContain('unavailable');
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
|
|
heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
|
|
expect(
|
|
pendingHostOperations(hostId).filter((op) => op.kind === 'instance.cancel-create')
|
|
).toHaveLength(1);
|
|
await acknowledge(cleanup);
|
|
expect((await detail(cleanup)).state).toBe('done');
|
|
});
|
|
|
|
it('replaces an occupancy-blocked removal and ignores an outstanding creation health reply', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { healthy: true }
|
|
});
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(
|
|
() =>
|
|
new Promise((resolve) => {
|
|
release = resolve;
|
|
})
|
|
);
|
|
const pending = advance();
|
|
try {
|
|
const { destroyServer } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
const waitingRemoval = destroyServer(job.name).deploymentId!;
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
expect(cleanup).toBeTruthy();
|
|
expect((await detail(waitingRemoval)).state).toBe('cancelled');
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await pending;
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect((await detail(cleanup)).state).toBe('running');
|
|
} finally {
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await pending;
|
|
}
|
|
});
|
|
|
|
it('does not offer creation cancellation after success or against a replacement instance', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { canCancelCreation, cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(canCancelCreation(job.deploymentId)).toBe(false);
|
|
expect(cancelCreation(job.deploymentId)).toBeNull();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?")
|
|
.run(job.deploymentId);
|
|
db()
|
|
.prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?')
|
|
.run(job.name);
|
|
expect(cancelCreation(job.deploymentId)).toBeNull();
|
|
});
|
|
|
|
it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => {
|
|
const job = await queue();
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { isProvisioningDetail, provisioningGroups } =
|
|
await import('../../src/lib/ui/provisioning-presentation.js');
|
|
const queued = getDeployment(job.deploymentId)!;
|
|
expect(queued.progress?.phase).toBe('eligibility');
|
|
expect(queued.progress?.blocker).toBeNull();
|
|
expect(queued.backend).toBe('native');
|
|
await advance();
|
|
expect(getDeployment(job.deploymentId)!.progress?.recipeStatus).toBe('attempted');
|
|
await acknowledge(job.deploymentId);
|
|
const result = getDeployment(job.deploymentId)!;
|
|
expect(result.deployment.state).toBe('done');
|
|
expect(isProvisioningDetail(result, job.deploymentId)).toBe(true);
|
|
expect(result.progress?.checks.map((check) => check.key)).toEqual([
|
|
'container',
|
|
'game',
|
|
'map',
|
|
'plutonium-release',
|
|
'plutonium-version'
|
|
]);
|
|
expect(result.progress?.checks.every((check) => check.result === 'pass')).toBe(true);
|
|
expect(result.progress?.checks[0].source).toBe('agent');
|
|
const steps = provisioningGroups(result).flatMap((group) => group.steps);
|
|
expect(steps.every((step) => step.state === 'done')).toBe(true);
|
|
expect(result.agentOperation?.state).toBe('completed');
|
|
});
|
|
|
|
it('records agent receipts immediately and only once, even before the provisioning worker runs', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
for (const state of ['accepted', 'accepted', 'running', 'completed', 'completed']) {
|
|
reportHostOperation(hostId, operation.id, {
|
|
state,
|
|
result: { state: 'running', healthy: true }
|
|
});
|
|
}
|
|
const result = getDeployment(job.deploymentId)!;
|
|
expect(result.deployment.state).toBe('running');
|
|
expect(result.agentOperation?.state).toBe('completed');
|
|
expect(result.events.filter((event) => event.kind === 'agent')).toHaveLength(3);
|
|
expect(result.progress?.checkpoints.map((point) => point.key)).toEqual(
|
|
expect.arrayContaining(['agent-accepted', 'agent-running', 'agent-completed'])
|
|
);
|
|
expect(result.progress?.command?.state).toBe('accepted');
|
|
});
|
|
|
|
it('records an outstanding container check before awaiting the host and admits other jobs meanwhile', async () => {
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const first = await queue();
|
|
await advance();
|
|
const operation = (await operationFor(first.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { healthy: true }
|
|
});
|
|
const second = await queue();
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(
|
|
() =>
|
|
new Promise((resolve) => {
|
|
release = resolve;
|
|
})
|
|
);
|
|
const pending = advance();
|
|
try {
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
expect(getDeployment(first.deploymentId)!.progress?.phase).toBe('verify-container');
|
|
await vi.waitFor(
|
|
async () => {
|
|
expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
|
|
},
|
|
{ timeout: 5_000 }
|
|
);
|
|
} finally {
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await pending;
|
|
}
|
|
});
|
|
|
|
it('admits a later request on the next scheduler pass while an earlier native probe is pending', async () => {
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const first = await queue();
|
|
await advance();
|
|
const operation = (await operationFor(first.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { healthy: true }
|
|
});
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(
|
|
() =>
|
|
new Promise((resolve) => {
|
|
release = resolve;
|
|
})
|
|
);
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning({ wait: false });
|
|
try {
|
|
const second = await queue();
|
|
await processProvisioning({ wait: false });
|
|
await vi.waitFor(
|
|
async () => {
|
|
expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
|
|
},
|
|
{ timeout: 5_000 }
|
|
);
|
|
expect(vi.mocked(nativeRequest)).toHaveBeenCalledTimes(1);
|
|
} finally {
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await processProvisioning();
|
|
}
|
|
});
|
|
|
|
it('ignores a late health result after removal retires the operation', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { healthy: true }
|
|
});
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(
|
|
() =>
|
|
new Promise((resolve) => {
|
|
release = resolve;
|
|
})
|
|
);
|
|
const pending = advance();
|
|
try {
|
|
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET state='cancelled',server_deleted_at=? WHERE id=?"
|
|
)
|
|
.run(new Date().toISOString(), job.deploymentId);
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const before = getDeployment(job.deploymentId)!;
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await pending;
|
|
const after = getDeployment(job.deploymentId)!;
|
|
expect(after.deployment.state).toBe('cancelled');
|
|
expect(after.progress).toEqual(before.progress);
|
|
expect(after.activity).toEqual(before.activity);
|
|
} finally {
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await pending;
|
|
}
|
|
});
|
|
|
|
it('records map mismatch evidence while retaining passed container and game checks', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
remote.map = 'zombie_moon';
|
|
await acknowledge(job.deploymentId);
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const result = getDeployment(job.deploymentId)!;
|
|
expect(result.deployment.state).toBe('awaiting-health');
|
|
expect(result.progress?.phase).toBe('verify-map');
|
|
expect(result.progress?.checks).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ key: 'container', result: 'pass' }),
|
|
expect.objectContaining({ key: 'game', result: 'pass' }),
|
|
expect.objectContaining({
|
|
key: 'map',
|
|
result: 'waiting',
|
|
actual: 'zombie_moon',
|
|
expected: 'zombie_theater'
|
|
})
|
|
])
|
|
);
|
|
});
|
|
|
|
it('selects the latest game files automatically and rejects stale explicit selections', async () => {
|
|
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
|
|
expect(() => nativeSelection({ hostId, image }, 't5')).toThrow('Upload T5 game files');
|
|
const old = await prepareLibrary('t5');
|
|
expect(nativeSelection({ hostId, image }, 't5').libraryId).toBe(old.id);
|
|
const latest = await prepareLibrary('t5');
|
|
expect(nativeSelection({ hostId, image }, 't5').libraryId).toBe(latest.id);
|
|
expect(() => nativeSelection({ hostId, image, libraryId: old.id }, 't5')).toThrow(
|
|
'Refresh the launch preview'
|
|
);
|
|
});
|
|
|
|
it('rejects a launch preview made before newer game files were published without reserving a server', async () => {
|
|
await prepareLibrary('t5');
|
|
const { previewNativeCreation, queueNativeCreation } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
const preview = previewNativeCreation({ hostId, image, game: 't5', mapSlug: 'theater' });
|
|
await prepareLibrary('t5');
|
|
const result = queueNativeCreation({
|
|
game: 't5',
|
|
gameMode: 'zombie',
|
|
mapSlug: 'theater',
|
|
previewId: preview.id,
|
|
plutoKey: 'new-key'
|
|
});
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
message: expect.stringContaining('Refresh the launch preview')
|
|
});
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({
|
|
count: 0
|
|
});
|
|
expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({
|
|
count: 0
|
|
});
|
|
});
|
|
|
|
it.each(['pinned', 'automatic'] as const)(
|
|
'keeps an unplaced %s launch pinned to its accepted files during cleanup',
|
|
async (placement) => {
|
|
const job = await queue('t5', placement);
|
|
const acceptedId = libraryId;
|
|
const latest = await prepareLibrary('t5');
|
|
expect(latest.id).not.toBe(acceptedId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const {
|
|
reconcileHostLibraryCleanup,
|
|
cleanupControllerLibraries,
|
|
getLibraryVersion,
|
|
updateHostLibraryCaches
|
|
} = await import('../../src/lib/server/game-files/index.js');
|
|
const accepted = getLibraryVersion(acceptedId)!;
|
|
heartbeat(hostId, { ...inventory(), capabilities: ['library.remove'] });
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
|
reconcileHostLibraryCleanup(hostId);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT id FROM ec_host_operations WHERE kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=?"
|
|
)
|
|
.all(acceptedId)
|
|
).toEqual([]);
|
|
|
|
// The unplaced intent also protects the controller's copy when every
|
|
// host reports it absent; its accepted version remains downloadable.
|
|
updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'missing' }]);
|
|
await cleanupControllerLibraries();
|
|
expect(
|
|
db()
|
|
.prepare('SELECT version_id FROM game_file_cleanup WHERE version_id=?')
|
|
.get(acceptedId)
|
|
).toBeUndefined();
|
|
const { stat } = await import('node:fs/promises');
|
|
expect(
|
|
(
|
|
await stat(join(fixture.root, 'libraries', 't5', acceptedId, 'zone/base.ff'))
|
|
).isFile()
|
|
).toBe(true);
|
|
updateHostLibraryCaches(hostId, [
|
|
{
|
|
libraryVersionId: acceptedId,
|
|
status: 'ready',
|
|
manifestSha256: accepted.manifestSha256
|
|
}
|
|
]);
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))?.payload.libraryId).toBe(acceptedId);
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.library_id).toBe(acceptedId);
|
|
}
|
|
);
|
|
|
|
it.each(['t5', 't6'] as const)(
|
|
'queues and verifies a %s deployment through its host agent',
|
|
async (game) => {
|
|
const job = await queue(game);
|
|
await advance();
|
|
const operation = await operationFor(job.deploymentId);
|
|
expect(job.port).toBeNull();
|
|
expect(operation).toMatchObject({
|
|
kind: 'instance.create',
|
|
hostId,
|
|
generation: 1,
|
|
state: 'pending',
|
|
payload: { game, libraryId, image, port: 28960 }
|
|
});
|
|
expect(operation?.payload.files).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ path: 'configs/electric_cherry_map.cfg' })
|
|
])
|
|
);
|
|
await acknowledge(job.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT protocol FROM ec_host_ports WHERE instance_id=? ORDER BY protocol'
|
|
)
|
|
.all(operation!.instanceId)
|
|
).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]);
|
|
}
|
|
);
|
|
it('reconciles the same operation after controller restart and an ambiguous remote result', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, original.id, {
|
|
state: 'unknown',
|
|
error: 'Reply lost after execution'
|
|
});
|
|
const { closeDb, db } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
const { interruptDeployments } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
interruptDeployments();
|
|
await advance();
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.create'"
|
|
)
|
|
.get()
|
|
).toEqual({ count: 1 });
|
|
expect((await detail(job.deploymentId)).state).toBe('running');
|
|
await acknowledge(job.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
});
|
|
it('holds disconnected instances as unknown without replacement and resumes the original operation', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?')
|
|
.run('2000-01-01T00:00:00Z', hostId);
|
|
await advance();
|
|
expect(
|
|
db().prepare('SELECT container_state FROM servers WHERE name=?').get(job.name)
|
|
).toEqual({ container_state: 'unknown' });
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, inventory());
|
|
await acknowledge(job.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
});
|
|
it('rebuilds only an empty instance and advances its generation after the acknowledged command', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test');
|
|
if (!rebuild.ok) throw new Error(rebuild.message);
|
|
remote.players = 2;
|
|
await advance();
|
|
expect(await operationFor(rebuild.deploymentId)).toBeNull();
|
|
remote.players = 0;
|
|
await advance();
|
|
const operation = (await operationFor(rebuild.deploymentId))!;
|
|
expect(operation).toMatchObject({
|
|
kind: 'instance.rebuild',
|
|
generation: 1,
|
|
payload: { nextGeneration: 2, libraryId }
|
|
});
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.generation).toBe(1);
|
|
await acknowledge(rebuild.deploymentId);
|
|
expect(nativeInstance(job.name)?.generation).toBe(2);
|
|
expect((await detail(rebuild.deploymentId)).state).toBe('done');
|
|
await advance();
|
|
expect(nativeInstance(job.name)?.generation).toBe(2);
|
|
});
|
|
it('carries confirmed admin override through serialized rebuild preparation', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), capabilities: ['instance.safety-override'] });
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test');
|
|
if (!rebuild.ok) throw new Error(rebuild.message);
|
|
remote.players = 2;
|
|
await advance();
|
|
expect(await operationFor(rebuild.deploymentId)).toBeNull();
|
|
const { enableSafetyOverride } =
|
|
await import('../../src/lib/server/provisioning/safety-override.js');
|
|
expect(
|
|
enableSafetyOverride(rebuild.deploymentId, { id: 'admin', username: 'admin' })
|
|
).toEqual({ ok: true });
|
|
await advance();
|
|
expect(await operationFor(rebuild.deploymentId)).toMatchObject({
|
|
kind: 'instance.rebuild',
|
|
generation: 1,
|
|
payload: { nextGeneration: 2, libraryId, safetyOverride: true }
|
|
});
|
|
await acknowledge(rebuild.deploymentId);
|
|
expect((await detail(rebuild.deploymentId)).state).toBe('done');
|
|
});
|
|
|
|
it('holds a rebuild when players join while its launch files are being prepared', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test');
|
|
if (!rebuild.ok) throw new Error(rebuild.message);
|
|
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
|
|
const prepare = preparation.prepareNativeLaunch;
|
|
let started!: () => void;
|
|
let release!: () => void;
|
|
const ready = new Promise<void>((resolve) => {
|
|
started = resolve;
|
|
});
|
|
const gate = new Promise<void>((resolve) => {
|
|
release = resolve;
|
|
});
|
|
const spy = vi
|
|
.spyOn(preparation, 'prepareNativeLaunch')
|
|
.mockImplementationOnce(async (input) => {
|
|
const result = await prepare(input);
|
|
started();
|
|
await gate;
|
|
return result;
|
|
});
|
|
const pending = advance();
|
|
try {
|
|
await ready;
|
|
remote.players = 2;
|
|
release();
|
|
await pending;
|
|
expect(await operationFor(rebuild.deploymentId)).toBeNull();
|
|
remote.players = 0;
|
|
await advance();
|
|
expect((await operationFor(rebuild.deploymentId))?.kind).toBe('instance.rebuild');
|
|
} finally {
|
|
release();
|
|
await pending;
|
|
spy.mockRestore();
|
|
}
|
|
});
|
|
it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test');
|
|
if (!rebuild.ok) throw new Error(rebuild.message);
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
const actual = await vi.importActual<
|
|
typeof import('../../src/lib/server/integrations/plutonium.js')
|
|
>('../../src/lib/server/integrations/plutonium.js');
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
|
|
// Version evidence must follow the submitted command, even on a fast test host.
|
|
db()
|
|
.prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?')
|
|
.run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
|
|
db()
|
|
.prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?')
|
|
.run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
|
|
await acknowledge(rebuild.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
expect((await detail(rebuild.deploymentId)).state).toBe('done');
|
|
expect(
|
|
db()
|
|
.prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?')
|
|
.get(rebuild.deploymentId)
|
|
).toEqual({ healthy_at: expect.any(String) });
|
|
expect(
|
|
actual.runningVersions().find((version) => version.server === job.name)?.revision
|
|
).toBe(5354);
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
|
|
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
|
|
expect(serverObservation(getServer(job.name)!).health).not.toBe('provisioning');
|
|
expect(
|
|
fleetOperations().filter(
|
|
(operation) => operation.server === job.name && operation.state !== 'done'
|
|
)
|
|
).toEqual([]);
|
|
});
|
|
it('resumes a rebuild waiting on version evidence after controller restart without rebuilding again', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test');
|
|
if (!rebuild.ok) throw new Error(rebuild.message);
|
|
await advance();
|
|
const operation = (await operationFor(rebuild.deploymentId))!;
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const actual = await vi.importActual<
|
|
typeof import('../../src/lib/server/integrations/plutonium.js')
|
|
>('../../src/lib/server/integrations/plutonium.js');
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
|
|
const submittedAt = new Date(Date.now() - 1000).toISOString();
|
|
db()
|
|
.prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?')
|
|
.run(submittedAt, rebuild.deploymentId);
|
|
db()
|
|
.prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?')
|
|
.run(submittedAt, rebuild.deploymentId);
|
|
vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({
|
|
revision: null,
|
|
error: 'Version response unavailable'
|
|
});
|
|
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
|
|
await acknowledge(rebuild.deploymentId);
|
|
expect(await detail(rebuild.deploymentId)).toMatchObject({
|
|
state: 'awaiting-health',
|
|
stage: 'Waiting to verify the Plutonium runtime revision.'
|
|
});
|
|
expect(
|
|
db()
|
|
.prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?')
|
|
.get(rebuild.deploymentId)
|
|
).toEqual({ healthy_at: null });
|
|
closeDb();
|
|
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
|
|
await advance();
|
|
expect((await detail(rebuild.deploymentId)).state).toBe('done');
|
|
expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'"
|
|
)
|
|
.get()
|
|
).toEqual({ count: 1 });
|
|
});
|
|
it('drains by reserving destination capacity, waiting for empty lobbies, and rebuilding once on another host', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const {
|
|
updateHost,
|
|
createHost,
|
|
createEnrollment,
|
|
enrollHost,
|
|
reportHostOperation,
|
|
getHost
|
|
} = await import('../../src/lib/server/hosts/index.js');
|
|
updateHost(hostId, { mode: 'draining' });
|
|
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
|
|
expect(() => nativeSelection({ hostId, libraryId, image }, 't5')).toThrow(/draining/);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { getLibraryVersion, updateHostLibraryCaches } =
|
|
await import('../../src/lib/server/game-files/index.js');
|
|
await processProvisioning();
|
|
const migration = db()
|
|
.prepare(
|
|
"SELECT id FROM server_deployments WHERE triggered_by='host-drain' AND server=?"
|
|
)
|
|
.get(job.name) as { id: string };
|
|
expect((await detail(migration.id)).stage).toContain('No host has capacity');
|
|
expect(
|
|
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()
|
|
).toEqual([]);
|
|
const target = createHost({ name: 'Destination', portMin: 29000, portMax: 29005 });
|
|
enrollHost(createEnrollment(target.id).token, inventory());
|
|
updateHostLibraryCaches(target.id, [
|
|
{
|
|
libraryVersionId: libraryId,
|
|
status: 'ready',
|
|
manifestSha256: getLibraryVersion(libraryId)!.manifestSha256
|
|
}
|
|
]);
|
|
remote.players = 2;
|
|
await processProvisioning();
|
|
expect((await detail(migration.id)).stage).toContain('2 player(s)');
|
|
expect(
|
|
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()
|
|
).toEqual([]);
|
|
remote.players = 0;
|
|
await processProvisioning();
|
|
const remove = db()
|
|
.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'")
|
|
.get() as { id: string };
|
|
expect(remove).toBeTruthy();
|
|
expect(
|
|
db().prepare('SELECT protocol FROM ec_host_ports WHERE host_id=?').all(target.id)
|
|
).toHaveLength(2);
|
|
expect(() => updateHost(target.id, { mode: 'draining' })).toThrow('incoming');
|
|
reportHostOperation(hostId, remove.id, { state: 'unknown' });
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
await processProvisioning();
|
|
expect(
|
|
db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)
|
|
).toEqual({ host_id: hostId });
|
|
reportHostOperation(hostId, remove.id, { state: 'completed' });
|
|
await processProvisioning();
|
|
expect(
|
|
db()
|
|
.prepare('SELECT host_id,generation FROM native_instances WHERE server=?')
|
|
.get(job.name)
|
|
).toEqual({ host_id: target.id, generation: 2 });
|
|
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]);
|
|
expect(getHost(hostId)?.mode).toBe('draining');
|
|
const { cancelProvisioning } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
expect(cancelProvisioning(migration.id)).toBe(false);
|
|
await processProvisioning();
|
|
const create = (await operationFor(migration.id))!;
|
|
expect(create).toMatchObject({
|
|
hostId: target.id,
|
|
kind: 'instance.create',
|
|
generation: 2,
|
|
payload: { name: job.name, port: 29000 }
|
|
});
|
|
reportHostOperation(target.id, create.id, {
|
|
state: 'completed',
|
|
result: { state: 'running', healthy: true }
|
|
});
|
|
await processProvisioning();
|
|
expect((await detail(migration.id)).state).toBe('done');
|
|
expect(getHost(hostId)?.mode).toBe('disabled');
|
|
expect(
|
|
db().prepare('SELECT generation FROM native_instances WHERE server=?').get(job.name)
|
|
).toEqual({ generation: 2 });
|
|
await advance();
|
|
expect((await operationFor(migration.id))?.id).toBe(create.id);
|
|
expect(
|
|
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()
|
|
).toHaveLength(1);
|
|
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
|
|
});
|
|
|
|
it('reserves migration capacity before source removal and retains it while a replacement awaits startup', async () => {
|
|
const first = await queue();
|
|
await advance();
|
|
await acknowledge(first.deploymentId);
|
|
const { queueCreateServer } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
const second = queueCreateServer({
|
|
game: 't5',
|
|
gameMode: 'zombie',
|
|
mapSlug: 'theater',
|
|
hostId,
|
|
libraryId,
|
|
image,
|
|
plutoKey: 'second-key'
|
|
});
|
|
if (!second.ok) throw new Error(second.message);
|
|
await advance();
|
|
await acknowledge(second.deploymentId);
|
|
const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const { getLibraryVersion, updateHostLibraryCaches } =
|
|
await import('../../src/lib/server/game-files/index.js');
|
|
const target = createHost({ name: 'One free slot' });
|
|
enrollHost(createEnrollment(target.id).token, {
|
|
...inventory(),
|
|
memoryTotal: 4 * 1024 ** 3,
|
|
memoryAvailable: 1.75 * 1024 ** 3
|
|
});
|
|
updateHostLibraryCaches(target.id, [
|
|
{
|
|
libraryVersionId: libraryId,
|
|
status: 'ready',
|
|
manifestSha256: getLibraryVersion(libraryId)!.manifestSha256
|
|
}
|
|
]);
|
|
updateHost(hostId, { mode: 'draining' });
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const removals = () =>
|
|
db()
|
|
.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'")
|
|
.all() as { id: string }[];
|
|
expect(removals()).toHaveLength(1);
|
|
expect(
|
|
db().prepare('SELECT instance_id FROM ec_host_ports WHERE host_id=?').all(target.id)
|
|
).toHaveLength(2);
|
|
const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
|
|
expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow(
|
|
'memory capacity'
|
|
);
|
|
reportHostOperation(hostId, removals()[0].id, { state: 'completed' });
|
|
await advance();
|
|
expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow(
|
|
'memory capacity'
|
|
);
|
|
expect(removals()).toHaveLength(1);
|
|
});
|
|
|
|
it('retries definite migration failures without replaying successful source removal', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const {
|
|
createHost,
|
|
createEnrollment,
|
|
enrollHost,
|
|
updateHost,
|
|
reportHostOperation,
|
|
getHost
|
|
} = await import('../../src/lib/server/hosts/index.js');
|
|
const { getLibraryVersion, updateHostLibraryCaches } =
|
|
await import('../../src/lib/server/game-files/index.js');
|
|
const target = createHost({ name: 'Destination' });
|
|
enrollHost(createEnrollment(target.id).token, inventory());
|
|
updateHostLibraryCaches(target.id, [
|
|
{
|
|
libraryVersionId: libraryId,
|
|
status: 'ready',
|
|
manifestSha256: getLibraryVersion(libraryId)!.manifestSha256
|
|
}
|
|
]);
|
|
updateHost(hostId, { mode: 'draining' });
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const migration = db()
|
|
.prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations')
|
|
.get() as { deployment_id: string; remove_operation_id: string };
|
|
const expireRetry = (id: string) =>
|
|
db()
|
|
.prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?')
|
|
.run(new Date(Date.now() - 61_000).toISOString(), id);
|
|
reportHostOperation(hostId, migration.remove_operation_id, {
|
|
state: 'failed',
|
|
error: 'Docker unavailable before removal'
|
|
});
|
|
await advance();
|
|
expect(
|
|
db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)
|
|
).toEqual({ host_id: hostId });
|
|
expireRetry(migration.remove_operation_id);
|
|
await advance();
|
|
await advance();
|
|
const retry = db().prepare('SELECT remove_operation_id FROM ec_host_migrations').get() as {
|
|
remove_operation_id: string;
|
|
};
|
|
expect(retry.remove_operation_id).not.toBe(migration.remove_operation_id);
|
|
reportHostOperation(hostId, retry.remove_operation_id, { state: 'completed' });
|
|
await advance();
|
|
await advance();
|
|
const creation = (await operationFor(migration.deployment_id))!;
|
|
reportHostOperation(target.id, creation.id, {
|
|
state: 'failed',
|
|
error: 'Docker unavailable before creation'
|
|
});
|
|
expireRetry(creation.id);
|
|
await advance();
|
|
await advance();
|
|
const replacement = (await operationFor(migration.deployment_id))!;
|
|
expect(replacement.id).not.toBe(creation.id);
|
|
expect(replacement).toMatchObject({
|
|
kind: 'instance.create',
|
|
hostId: target.id,
|
|
generation: 2
|
|
});
|
|
reportHostOperation(target.id, replacement.id, { state: 'completed' });
|
|
await advance();
|
|
expect(getHost(hostId)?.mode).toBe('disabled');
|
|
expect(
|
|
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()
|
|
).toHaveLength(2);
|
|
});
|
|
it('disables an empty draining host and redirects its unplaced launches to another host', async () => {
|
|
const job = await queue();
|
|
const { createHost, createEnrollment, enrollHost, updateHost, getHost } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const { getLibraryVersion, updateHostLibraryCaches } =
|
|
await import('../../src/lib/server/game-files/index.js');
|
|
const target = createHost({ name: 'Destination' });
|
|
enrollHost(createEnrollment(target.id).token, inventory());
|
|
updateHostLibraryCaches(target.id, [
|
|
{
|
|
libraryVersionId: libraryId,
|
|
status: 'ready',
|
|
manifestSha256: getLibraryVersion(libraryId)!.manifestSha256
|
|
}
|
|
]);
|
|
updateHost(hostId, { mode: 'draining' });
|
|
expect(() => updateHost(hostId, { mode: 'active' })).toThrow('finish draining');
|
|
await advance();
|
|
expect(getHost(hostId)?.mode).toBe('disabled');
|
|
expect(await operationFor(job.deploymentId)).toMatchObject({
|
|
hostId: target.id,
|
|
kind: 'instance.create'
|
|
});
|
|
});
|
|
|
|
it('rejects the removed manual disable API action', async () => {
|
|
const { PATCH } = await import('../../src/routes/api/hosts/[id]/+server.js');
|
|
const response = await PATCH({
|
|
params: { id: hostId },
|
|
request: new Request(`https://ec.test/api/hosts/${hostId}`, {
|
|
method: 'PATCH',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ mode: 'disabled' })
|
|
})
|
|
});
|
|
expect(response.status).toBe(400);
|
|
expect(await response.json()).toMatchObject({
|
|
error: expect.stringContaining('disable automatically')
|
|
});
|
|
const { getHost } = await import('../../src/lib/server/hosts/index.js');
|
|
expect(getHost(hostId)?.mode).toBe('active');
|
|
});
|
|
it('builds campaign replacements with requested plugins, configuration, additions and removals', async () => {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
for (const path of ['replace.dll', 'remove.dll']) {
|
|
const file = createScript({
|
|
name: path,
|
|
kind: 'plugin',
|
|
targetRelPath: path,
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content: Buffer.from('promoted plugin')
|
|
});
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(file.version.id, file.script.id);
|
|
}
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO deployments(id,action,started_at) VALUES('files-rebuild','promote',?)"
|
|
)
|
|
.run(new Date().toISOString());
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('files-rebuild',?,4,'pending')"
|
|
)
|
|
.run(job.name);
|
|
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
const changes = [
|
|
{ kind: 'plugin', relPath: 'replace.dll', content: Buffer.from('requested plugin') },
|
|
{ kind: 'plugin', relPath: 'remove.dll', content: null },
|
|
{ kind: 'script', relPath: 'new.gsc', content: Buffer.from('new script') },
|
|
{ kind: 'config', relPath: 'custom.cfg', content: Buffer.from('set custom 1') }
|
|
] as const;
|
|
for (const change of changes)
|
|
queueFileChange({
|
|
sourceKind: 'campaign',
|
|
sourceId: 'files-rebuild',
|
|
server: job.name,
|
|
inventorySource: 'campaign',
|
|
...change
|
|
});
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const rebuild = requestRestart(job.name, { kind: 'campaign', id: 'files-rebuild' });
|
|
await advance();
|
|
const { getHostOperation, reportHostOperation } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const staged = db()
|
|
.prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
|
|
.get(rebuild.id) as { stage_operation_id: string };
|
|
reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
|
|
await advance();
|
|
const operation = (await operationFor(rebuild.id))!;
|
|
expect(operation.kind).toBe('instance.rebuild');
|
|
expect(operation.payload).not.toHaveProperty('unhealthyRecovery');
|
|
const { recipeForDeployment } =
|
|
await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const recipe = recipeForDeployment(rebuild.id)!.recipe;
|
|
for (const change of changes) {
|
|
const actual = recipe.files.find(
|
|
(file) => file.kind === change.kind && file.path === change.relPath
|
|
);
|
|
if (change.content === null) expect(actual).toBeUndefined();
|
|
else expect(Buffer.from(actual!.content!, 'base64')).toEqual(change.content);
|
|
}
|
|
const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{
|
|
id: number;
|
|
path: string;
|
|
}>;
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { before: files.map((file) => ({ ...file, contentBase64: null })) }
|
|
});
|
|
remote.state = 'unhealthy';
|
|
await advance();
|
|
expect((await operationFor(rebuild.id))!.kind).toBe('instance.restart');
|
|
await acknowledge(rebuild.id);
|
|
const recovery = (await operationFor(rebuild.id))!;
|
|
expect(recovery.kind).toBe('instance.rebuild');
|
|
expect(recovery.payload).not.toHaveProperty('stageId');
|
|
expect(recovery.payload.files).toEqual(operation.payload.files);
|
|
remote.state = 'running';
|
|
await acknowledge(rebuild.id);
|
|
expect((await detail(rebuild.id)).state).toBe('done');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.generation).toBe(3);
|
|
});
|
|
|
|
it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const before = Buffer.from('original bytes on the host');
|
|
const after = Buffer.from('new campaign bytes');
|
|
db()
|
|
.prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)")
|
|
.run(new Date().toISOString());
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')"
|
|
)
|
|
.run(job.name);
|
|
const fileId = Number(
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after_hash,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)"
|
|
)
|
|
.run(job.name, putFileBlob(after, db())).lastInsertRowid
|
|
);
|
|
const { queueFileChange, stagedFileChanges } =
|
|
await import('../../src/lib/server/fleet/staged-files.js');
|
|
queueFileChange({
|
|
sourceKind: 'campaign',
|
|
sourceId: 'campaign',
|
|
server: job.name,
|
|
kind: 'script',
|
|
relPath: 'hud.gsc',
|
|
content: after,
|
|
inventorySource: 'test',
|
|
deploymentFileId: fileId
|
|
});
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const restart = requestRestart(job.name, { kind: 'campaign', id: 'campaign' });
|
|
expect(restart.action).toBe('rebuild');
|
|
// Upgrade queued work from older controllers, but wait for a capable host.
|
|
db().prepare("UPDATE server_deployments SET action='restart' WHERE id=?").run(restart.id);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), capabilities: [] });
|
|
await advance();
|
|
expect(await detail(restart.id)).toMatchObject({
|
|
action: 'rebuild',
|
|
state: 'waiting',
|
|
stage: expect.stringContaining('Update the host agent')
|
|
});
|
|
expect(await operationFor(restart.id)).toBeNull();
|
|
heartbeat(hostId, inventory());
|
|
remote.players = 2;
|
|
await advance();
|
|
const staged = db()
|
|
.prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
|
|
.get(restart.id) as { stage_operation_id: string };
|
|
expect(staged.stage_operation_id).toBeTruthy();
|
|
const { getHostOperation, reportHostOperation } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const stage = getHostOperation(staged.stage_operation_id)!;
|
|
expect(stage.kind).toBe('files.stage');
|
|
expect(await operationFor(restart.id)).toBeNull();
|
|
const routing = db()
|
|
.prepare('SELECT payload FROM ec_host_operations WHERE id=?')
|
|
.get(stage.id) as { payload: string };
|
|
expect(JSON.parse(routing.payload)).toEqual({ stageId: restart.id });
|
|
const plan = db()
|
|
.prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?')
|
|
.get(restart.id) as { files_json: string };
|
|
expect(JSON.parse(plan.files_json)[0]).not.toHaveProperty('contentBase64');
|
|
const storedPayload = db()
|
|
.prepare('SELECT payload FROM ec_host_operation_payloads WHERE operation_id=?')
|
|
.get(stage.id) as { payload: string };
|
|
expect(JSON.parse(storedPayload.payload).files[0]).toMatchObject({
|
|
contentSha256: createHash('sha256').update(after).digest('hex')
|
|
});
|
|
expect(JSON.parse(storedPayload.payload).files[0]).not.toHaveProperty('contentBase64');
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
const { pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
|
|
expect(
|
|
pendingHostOperations(hostId).find((operation) => operation.id === stage.id)?.payload
|
|
).toEqual(stage.payload);
|
|
await advance();
|
|
expect(
|
|
(
|
|
db()
|
|
.prepare(
|
|
'SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?'
|
|
)
|
|
.get(restart.id) as { stage_operation_id: string }
|
|
).stage_operation_id
|
|
).toBe(stage.id);
|
|
expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('pending');
|
|
reportHostOperation(hostId, stage.id, { state: 'completed', result: { state: 'staged' } });
|
|
await advance();
|
|
expect(await operationFor(restart.id)).toBeNull();
|
|
remote.players = 0;
|
|
await advance();
|
|
const activation = (await operationFor(restart.id))!;
|
|
expect(activation).toMatchObject({
|
|
kind: 'instance.rebuild',
|
|
generation: 1,
|
|
payload: {
|
|
stageId: restart.id,
|
|
nextGeneration: 2,
|
|
files: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
path: 'runtime/plutonium/storage/t5/raw/scripts/sp/hud.gsc',
|
|
sha256: createHash('sha256').update(after).digest('hex')
|
|
})
|
|
])
|
|
}
|
|
});
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,snapshot_pending FROM deployment_files WHERE id=?'
|
|
)
|
|
.get(fileId)
|
|
).toEqual({ content_before: null, snapshot_pending: 1 });
|
|
const files = stage.payload.files as Array<{ id: number; path: string }>;
|
|
reportHostOperation(hostId, activation.id, {
|
|
state: 'completed',
|
|
result: {
|
|
before: [
|
|
{
|
|
id: files[0].id,
|
|
path: files[0].path,
|
|
contentBase64: before.toString('base64')
|
|
}
|
|
]
|
|
}
|
|
});
|
|
await advance();
|
|
expect((await detail(restart.id)).state).toBe('done');
|
|
expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied');
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?'
|
|
)
|
|
.get(fileId)
|
|
).toEqual({
|
|
content_before: before,
|
|
sha256_before: createHash('sha256').update(before).digest('hex'),
|
|
snapshot_pending: 0
|
|
});
|
|
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(
|
|
createHash('sha256').update(after).digest('hex')
|
|
);
|
|
});
|
|
it('atomically queues mixed-game collections and reserves per-host ports when dispatched', async () => {
|
|
const t5 = await prepareLibrary('t5');
|
|
const t6 = await prepareLibrary('t6');
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const blueprint = saveBlueprint({
|
|
name: 'Both games',
|
|
entries: [
|
|
{ game: 't5', mapSlug: 'theater', count: 2 },
|
|
{ game: 't6', mapSlug: 'town', count: 1 }
|
|
]
|
|
});
|
|
const { previewCollection, queueCollection } =
|
|
await import('../../src/lib/server/fleet/collections.js');
|
|
const preview = previewCollection(blueprint.id, 'fill', {
|
|
hostId,
|
|
libraryIds: { t5: t5.id, t6: t6.id },
|
|
image
|
|
});
|
|
expect(preview.ready).toBe(true);
|
|
expect(preview.servers.map((server) => server.port)).toEqual([null, null, null]);
|
|
const queued = queueCollection({
|
|
previewId: preview.id,
|
|
keys: Object.fromEntries(
|
|
preview.servers.map((server) => [server.name!, 'key-' + server.name])
|
|
)
|
|
});
|
|
expect(queueCollection({ previewId: preview.id }).collectionId).toBe(queued.collectionId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 });
|
|
await advance();
|
|
const instances = db()
|
|
.prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port')
|
|
.all();
|
|
expect(instances).toEqual([
|
|
{ host_id: hostId, library_id: t5.id, port: 28960 },
|
|
{ host_id: hostId, library_id: t5.id, port: 28961 },
|
|
{ host_id: hostId, library_id: t6.id, port: 28962 }
|
|
]);
|
|
const { createHost, createEnrollment, enrollHost, hostPortAvailable } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const secondHost = createHost({ name: 'Second Debian' });
|
|
enrollHost(createEnrollment(secondHost.id).token, inventory());
|
|
const { updateHostLibraryCaches } =
|
|
await import('../../src/lib/server/game-files/index.js');
|
|
updateHostLibraryCaches(secondHost.id, [
|
|
{ libraryVersionId: t5.id, status: 'ready', manifestSha256: t5.manifestSha256 }
|
|
]);
|
|
const { queueCreateServer } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
const second = queueCreateServer({
|
|
game: 't5',
|
|
gameMode: 'zombie',
|
|
mapSlug: 'moon',
|
|
hostId: secondHost.id,
|
|
libraryId: t5.id,
|
|
image,
|
|
plutoKey: 'second-host-key'
|
|
});
|
|
expect(second.ok).toBe(true);
|
|
if (!second.ok) throw new Error(second.message);
|
|
expect(second.port).toBeNull();
|
|
await advance();
|
|
expect((await operationFor(second.deploymentId))?.payload.port).toBe(28960);
|
|
expect(hostPortAvailable(hostId, 28960)).toBe(false);
|
|
expect(hostPortAvailable(secondHost.id, 28960)).toBe(false);
|
|
expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({
|
|
count: 8
|
|
});
|
|
});
|
|
it('reads remote instance files through the agent including missing files and epoch timestamps', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
const remoteRead = vi.mocked(backend.nativeRequest);
|
|
remoteRead.mockImplementation(async (_instance, kind, payload) => {
|
|
if (payload?.path === 'missing')
|
|
return (kind === 'files.read' ? { contentBase64: null } : null) as never;
|
|
if (kind === 'files.stat')
|
|
return {
|
|
size: 15,
|
|
mtime: 1_790_000_000,
|
|
isFile: true,
|
|
isDirectory: false
|
|
} as never;
|
|
if (kind === 'files.read')
|
|
return {
|
|
contentBase64: Buffer.from('agent file bytes').toString('base64')
|
|
} as never;
|
|
if (kind === 'files.list')
|
|
return {
|
|
entries: [{ name: 'configs', isDirectory: true, isFile: false }]
|
|
} as never;
|
|
throw new Error('Unexpected file operation');
|
|
});
|
|
const files = await import('../../src/lib/server/backends/files.js');
|
|
const root = join(fixture.t5, job.name);
|
|
expect((await files.readFile(join(root, 'configs/server.cfg'))).toString()).toBe(
|
|
'agent file bytes'
|
|
);
|
|
const info = await files.stat(join(root, 'configs/server.cfg'));
|
|
expect(info.mtime.getTime()).toBe(1_790_000_000_000);
|
|
expect(info.isFile()).toBe(true);
|
|
expect(await files.exists(join(root, 'missing'))).toBe(false);
|
|
await expect(files.readFile(join(root, 'missing'))).rejects.toMatchObject({
|
|
code: 'ENOENT'
|
|
});
|
|
const entries = await files.readdir(root, { withFileTypes: true });
|
|
expect(entries[0].name).toBe('configs');
|
|
expect(entries[0].isDirectory()).toBe(true);
|
|
expect(remoteRead).toHaveBeenCalledWith(
|
|
expect.objectContaining({ server: job.name }),
|
|
'files.read',
|
|
{ path: 'configs/server.cfg' }
|
|
);
|
|
const { stat } = await import('node:fs/promises');
|
|
await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' });
|
|
});
|
|
it('retires an agent-confirmed missing container without waiting on nonexistent RCON', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const removal = destroyServer(job.name);
|
|
if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconStatusResult).mockClear();
|
|
remote.state = 'missing';
|
|
await advance();
|
|
expect((await operationFor(removal.deploymentId))?.kind).toBe('instance.remove');
|
|
expect(rconStatusResult).not.toHaveBeenCalled();
|
|
await acknowledge(removal.deploymentId);
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
expect((await detail(removal.deploymentId)).state).toBe('done');
|
|
});
|
|
it('releases a failed activation slot without recording fictitious file changes', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO deployments(id,action,started_at) VALUES('preflight-failed','promote',?)"
|
|
)
|
|
.run(new Date().toISOString());
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('preflight-failed',?,1,'pending')"
|
|
)
|
|
.run(job.name);
|
|
const fileId = Number(
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO deployment_files(deployment_id,server,verb,rel_path) VALUES('preflight-failed',?,'updated','hud.gsc')"
|
|
)
|
|
.run(job.name).lastInsertRowid
|
|
);
|
|
const { queueFileChange, stagedFileChanges } =
|
|
await import('../../src/lib/server/fleet/staged-files.js');
|
|
queueFileChange({
|
|
sourceKind: 'campaign',
|
|
sourceId: 'preflight-failed',
|
|
server: job.name,
|
|
kind: 'script',
|
|
relPath: 'hud.gsc',
|
|
content: Buffer.from('new bytes'),
|
|
inventorySource: 'test',
|
|
deploymentFileId: fileId
|
|
});
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const restart = requestRestart(job.name, { kind: 'campaign', id: 'preflight-failed' });
|
|
await advance();
|
|
const stage = db()
|
|
.prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
|
|
.get(restart.id) as { stage_operation_id: string };
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, stage.stage_operation_id, {
|
|
state: 'completed',
|
|
result: { state: 'staged' }
|
|
});
|
|
await advance();
|
|
const activation = (await operationFor(restart.id))!;
|
|
expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('applying');
|
|
reportHostOperation(hostId, activation.id, {
|
|
state: 'failed',
|
|
error: 'Target changed after campaign review; no activation occurred'
|
|
});
|
|
await advance();
|
|
expect((await detail(restart.id)).state).toBe('failed');
|
|
expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('pending');
|
|
expect(
|
|
db()
|
|
.prepare('SELECT COUNT(*) AS count FROM restart_rollout WHERE healthy_at IS NULL')
|
|
.get()
|
|
).toEqual({ count: 0 });
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before FROM deployment_files WHERE id=?'
|
|
)
|
|
.get(fileId)
|
|
).toEqual({ content_before: null, sha256_before: null });
|
|
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined();
|
|
});
|
|
it('cancels an unplaced native launch locally without reserving ports or submitting a create', async () => {
|
|
const job = await queue();
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const cancelled = destroyServer(job.name);
|
|
expect(cancelled.ok).toBe(true);
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'"
|
|
)
|
|
.get()
|
|
).toEqual({ count: 0 });
|
|
const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
|
|
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
|
|
const { getServerKey, serverKeyPool } =
|
|
await import('../../src/lib/server/fleet/server-keys.js');
|
|
expect(getServerKey(job.name)).toBeUndefined();
|
|
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
|
|
});
|
|
});
|
|
|
|
describe('automatic capacity admission and placement', () => {
|
|
const input = {
|
|
game: 't5' as const,
|
|
gameMode: 'zombie' as const,
|
|
mapSlug: 'theater',
|
|
plutoKey: 'key'
|
|
};
|
|
async function secondHost(overrides = {}) {
|
|
const api = await import('../../src/lib/server/hosts/index.js');
|
|
const libs = await import('../../src/lib/server/game-files/index.js');
|
|
const host = api.createHost({
|
|
name: 'Second host',
|
|
publicAddress: 'second.example',
|
|
portMin: 28960,
|
|
portMax: 28970
|
|
});
|
|
api.updateHost(host.id, { publicAddress: 'second.example' });
|
|
api.enrollHost(api.createEnrollment(host.id).token, { ...inventory(), ...overrides });
|
|
const version = libs.getLibraryVersion(libraryId)!;
|
|
libs.updateHostLibraryCaches(host.id, [
|
|
{ libraryVersionId: libraryId, status: 'ready', manifestSha256: version.manifestSha256 }
|
|
]);
|
|
return host;
|
|
}
|
|
it('admits without a host, then selects the least loaded eligible host and its network defaults', async () => {
|
|
await prepareLibrary();
|
|
const other = await secondHost({ cpuUsagePercent: 0 });
|
|
const { queueCreateServer } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const { recipeForDeployment } =
|
|
await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const result = queueCreateServer(input);
|
|
expect(result.ok).toBe(true);
|
|
if (!result.ok) return;
|
|
expect(nativeInstance(result.name)).toBeNull();
|
|
expect(result.port).toBeNull();
|
|
await advance();
|
|
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
|
|
expect(nativeInstance(result.name)).toMatchObject({
|
|
host_id: other.id,
|
|
public_address: 'second.example'
|
|
});
|
|
expect(recipeForDeployment(result.deploymentId)?.recipe.native?.hostId).toBe(other.id);
|
|
// Once submitted, even a now-offline host cannot trigger another placement.
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(other.id);
|
|
await advance();
|
|
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
|
|
expect(
|
|
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()
|
|
).toHaveLength(1);
|
|
});
|
|
it('moves an unsubmitted reservation off an offline host and preserves explicit network overrides', async () => {
|
|
await prepareLibrary();
|
|
await secondHost();
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const result = queueNativeCreation({
|
|
...input,
|
|
publicAddress: 'nat.example',
|
|
publicPort: 30000
|
|
});
|
|
expect(result.ok).toBe(true);
|
|
if (!result.ok) return;
|
|
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
|
|
await advance();
|
|
expect(nativeInstance(result.name)).toMatchObject({
|
|
public_address: 'nat.example',
|
|
public_port: 30000
|
|
});
|
|
expect((await operationFor(result.deploymentId))?.hostId).not.toBe(hostId);
|
|
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]);
|
|
});
|
|
it.each([
|
|
{ cpuUsagePercent: undefined },
|
|
{ memoryAvailable: undefined },
|
|
{ cpuCount: 0 },
|
|
{ cpuUsagePercent: 99 },
|
|
{ memoryAvailable: 1024 ** 3 }
|
|
])('accepts an unplaced job and waits for unavailable resources: %j', async (overrides) => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), ...overrides });
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const queued = queueNativeCreation(input);
|
|
expect(queued).toMatchObject({ ok: true, port: null });
|
|
if (!queued.ok) throw new Error(queued.message);
|
|
await advance();
|
|
expect(await operationFor(queued.deploymentId)).toBeNull();
|
|
expect((await detail(queued.deploymentId)).stage).toContain('capacity');
|
|
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
});
|
|
it('rechecks preview admission without binding the request to the preview host or port', async () => {
|
|
await prepareLibrary();
|
|
const other = await secondHost();
|
|
const { updateHost } = await import('../../src/lib/server/hosts/index.js');
|
|
updateHost(other.id, { portMin: 30000, portMax: 30002 });
|
|
const { previewNativeCreation, queueNativeCreation } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
const preview = previewNativeCreation(input);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
|
|
const result = queueNativeCreation({ ...input, previewId: preview.id });
|
|
expect(result).toMatchObject({ ok: true, name: preview.name, port: null });
|
|
expect(queueNativeCreation({ ...input, previewId: preview.id })).toEqual(result);
|
|
if (!result.ok) throw new Error(result.message);
|
|
await advance();
|
|
expect((await operationFor(result.deploymentId))?.payload.port).toBe(30000);
|
|
});
|
|
it('queues the whole collection and waits when capacity shrinks after its preview', async () => {
|
|
await prepareLibrary();
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection } =
|
|
await import('../../src/lib/server/fleet/collections.js');
|
|
const blueprint = saveBlueprint({
|
|
name: 'Two lobbies',
|
|
entries: [{ game: 't5', mapSlug: 'theater', count: 2 }]
|
|
});
|
|
const preview = previewCollection(blueprint.id);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
|
|
const result = queueCollection({
|
|
previewId: preview.id,
|
|
keys: Object.fromEntries(
|
|
preview.servers.map((server) => [server.name!, 'key-' + server.name])
|
|
)
|
|
});
|
|
expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toHaveLength(1);
|
|
expect(db().prepare('SELECT * FROM native_creation_intents').all()).toHaveLength(2);
|
|
});
|
|
it.each([
|
|
{
|
|
resource: 'CPU',
|
|
measurements: { cpuCount: 4, cpuUsagePercent: 82.5 },
|
|
over: { cpuUsagePercent: 82.51 }
|
|
},
|
|
{
|
|
resource: 'memory',
|
|
measurements: { memoryAvailable: 1024 ** 3 + 500_000_000 },
|
|
over: { memoryAvailable: 1024 ** 3 + 499_999_999 }
|
|
}
|
|
])('reserves only the requested host $resource headroom', async ({ measurements, over }) => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
|
|
const options = { hostId, libraryId, image };
|
|
heartbeat(hostId, { ...inventory(), ...measurements });
|
|
// Includes the new server's 0.5-core / 1-GiB bootstrap reservation.
|
|
expect(choosePlacement(options).host.id).toBe(hostId);
|
|
expect(choosePlacement({ ...options, thorough: true }).host.id).toBe(hostId);
|
|
heartbeat(hostId, { ...inventory(), ...measurements, ...over });
|
|
expect(() => choosePlacement(options)).toThrow('capacity');
|
|
expect(() => choosePlacement({ ...options, thorough: true })).toThrow('capacity');
|
|
});
|
|
it('dispatches a waiting instance after calibration lowers the cold-start allowance', async () => {
|
|
const first = await queue();
|
|
await advance();
|
|
await acknowledge(first.deploymentId);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const { recordInstanceResources, capacityCalibration } =
|
|
await import('../../src/lib/server/hosts/calibration.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 50 };
|
|
heartbeat(hostId, measured);
|
|
const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
|
|
expect(next.ok).toBe(true);
|
|
if (!next.ok) throw new Error(next.message);
|
|
await advance();
|
|
expect(await operationFor(next.deploymentId)).toBeNull();
|
|
const instance = nativeInstance(first.name)!;
|
|
for (let minute = 0; minute < 25; minute++) {
|
|
recordInstanceResources(
|
|
hostId,
|
|
{
|
|
...measured,
|
|
instanceResources: [
|
|
{
|
|
instanceId: instance.id,
|
|
generation: 1,
|
|
uptimeSeconds: minute < 4 ? minute * 60 : 600,
|
|
cpuCores: 0.1,
|
|
memoryBytes: 256 * 1024 ** 2
|
|
}
|
|
]
|
|
},
|
|
new Date(Date.now() - (25 - minute) * 60_000).toISOString()
|
|
);
|
|
}
|
|
expect(capacityCalibration(hostId).startup.source).toBe('host');
|
|
await advance();
|
|
expect(await operationFor(next.deploymentId)).not.toBeNull();
|
|
});
|
|
it('queues a collection at running demand and releases startup reservations as live usage arrives', async () => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection } =
|
|
await import('../../src/lib/server/fleet/collections.js');
|
|
const GiB = 1024 ** 3;
|
|
const measured = {
|
|
...inventory(),
|
|
cpuCount: 4,
|
|
cpuUsagePercent: 5,
|
|
memoryTotal: 4 * GiB,
|
|
memoryAvailable: 3.75 * GiB
|
|
};
|
|
heartbeat(hostId, measured);
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
|
|
const insert = db().prepare(`INSERT INTO ec_resource_allowances
|
|
(scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,30,?,?,?)`);
|
|
const now = new Date().toISOString();
|
|
const start = new Date(Date.now() - 30 * 60_000).toISOString();
|
|
insert.run(hostId, 'startup', 2, 2 * GiB, start, now, now);
|
|
insert.run(hostId, 'running', 0.2, GiB / 4, start, now, now);
|
|
const blueprint = saveBlueprint({
|
|
name: 'Staggered lobbies',
|
|
entries: [{ game: 't5', mapSlug: 'theater', count: 3 }]
|
|
});
|
|
const preview = previewCollection(blueprint.id);
|
|
expect(preview.ready).toBe(true);
|
|
expect(
|
|
queueCollection({
|
|
previewId: preview.id,
|
|
keys: Object.fromEntries(
|
|
preview.servers.map((server) => [server.name!, 'key-' + server.name])
|
|
)
|
|
}).ok
|
|
).toBe(true);
|
|
await advance();
|
|
const creates = () =>
|
|
db()
|
|
.prepare("SELECT instance_id FROM ec_host_operations WHERE kind='instance.create'")
|
|
.all() as { instance_id: string }[];
|
|
expect(creates()).toHaveLength(1);
|
|
await advance();
|
|
expect(creates()).toHaveLength(1); // No heartbeat yet: the first launch still reserves startup resources.
|
|
heartbeat(hostId, {
|
|
...measured,
|
|
cpuUsagePercent: 10,
|
|
memoryAvailable: 3.5 * GiB,
|
|
instanceResources: [
|
|
{
|
|
instanceId: creates()[0].instance_id,
|
|
generation: 1,
|
|
uptimeSeconds: 60,
|
|
cpuCores: 0.2,
|
|
memoryBytes: GiB / 4
|
|
}
|
|
]
|
|
});
|
|
await advance();
|
|
expect(creates()).toHaveLength(2); // Its actual usage replaces the peak even before health verification.
|
|
});
|
|
it('reserves queued lobby identities without host resources and releases them on cancellation', async () => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true);
|
|
if (!first.ok) return;
|
|
const second = queueNativeCreation({ ...input, plutoKey: 'second-key' });
|
|
expect(second).toMatchObject({ ok: true, name: 't5_zombie_theater_2', port: null });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
cancelCreation(first.deploymentId);
|
|
expect(queueNativeCreation(input)).toMatchObject({
|
|
ok: true,
|
|
name: first.name,
|
|
port: null
|
|
});
|
|
});
|
|
it.each(['memory', 'cpu'] as const)(
|
|
'dispatches immediately after %s recovers instead of waiting for five-minute history',
|
|
async (resource) => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const GiB = 1024 ** 3;
|
|
// A recently busy 4 GiB host is now mostly empty, as in the reported
|
|
// ec-2 inventory. Admission must not pass only to stall in dispatch.
|
|
heartbeat(hostId, {
|
|
...inventory(),
|
|
cpuCount: 4,
|
|
memoryTotal: 4 * GiB,
|
|
memoryAvailable: Math.floor(3.8 * GiB)
|
|
});
|
|
for (let minute = 1; minute <= 4; minute++)
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO ec_host_resource_samples
|
|
(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,4,?,?,?)`
|
|
)
|
|
.run(
|
|
hostId,
|
|
new Date(Date.now() - minute * 60_000).toISOString(),
|
|
resource === 'cpu' ? 100 : 10,
|
|
4 * GiB,
|
|
resource === 'memory' ? GiB / 8 : Math.floor(3.8 * GiB)
|
|
);
|
|
const jobs = Array.from({ length: 3 }, (_, index) =>
|
|
queueNativeCreation({ ...input, plutoKey: `key-${index}` })
|
|
);
|
|
expect(jobs.every((job) => job.ok)).toBe(true);
|
|
await advance();
|
|
expect(
|
|
await operationFor(jobs[0].ok ? jobs[0].deploymentId : 'missing')
|
|
).not.toBeNull();
|
|
}
|
|
);
|
|
it('dispatches one queued launch when learned startup costs outgrow the original batch reservation', async () => {
|
|
await prepareLibrary();
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const GiB = 1024 ** 3;
|
|
heartbeat(hostId, {
|
|
...inventory(),
|
|
cpuCount: 4,
|
|
memoryTotal: 4 * GiB,
|
|
memoryAvailable: Math.floor(3.8 * GiB)
|
|
});
|
|
const jobs = Array.from({ length: 3 }, (_, index) =>
|
|
queueNativeCreation({ ...input, plutoKey: `key-${index}` })
|
|
);
|
|
expect(jobs.every((job) => job.ok)).toBe(true);
|
|
// Learning increases the startup allowance after admission. All three
|
|
// no longer fit concurrently, but this otherwise empty host fits one.
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO ec_resource_allowances
|
|
(scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at)
|
|
VALUES(?,'startup',0.5,?,3,?,?,?)`
|
|
)
|
|
.run(
|
|
hostId,
|
|
2 * GiB,
|
|
new Date(Date.now() - 180_000).toISOString(),
|
|
new Date().toISOString(),
|
|
new Date().toISOString()
|
|
);
|
|
await advance();
|
|
const operations = await Promise.all(
|
|
jobs.map((job) => operationFor(job.ok ? job.deploymentId : 'missing'))
|
|
);
|
|
expect(operations.filter(Boolean)).toHaveLength(1);
|
|
expect(operations[0]?.kind).toBe('instance.create');
|
|
// The dispatched operation now consumes capacity and blocks the others,
|
|
// even though the next host heartbeat has not observed it yet.
|
|
await advance();
|
|
expect(
|
|
db().prepare("SELECT * FROM ec_host_operations WHERE kind='instance.create'").all()
|
|
).toHaveLength(1);
|
|
const later = queueNativeCreation(input);
|
|
expect(later.ok).toBe(true);
|
|
if (!later.ok) throw new Error(later.message);
|
|
await advance();
|
|
expect(await operationFor(later.deploymentId)).toBeNull();
|
|
});
|
|
it('does not charge a starting container twice when its usage is already in the heartbeat', async () => {
|
|
await prepareLibrary();
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const GiB = 1024 ** 3;
|
|
const host = {
|
|
...inventory(),
|
|
cpuCount: 4,
|
|
memoryTotal: 3 * GiB,
|
|
memoryAvailable: 2.8 * GiB
|
|
};
|
|
heartbeat(hostId, { ...host, memoryAvailable: Math.floor(host.memoryAvailable) });
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true);
|
|
if (!first.ok) return;
|
|
await advance();
|
|
const instance = nativeInstance(first.name)!;
|
|
// 0.2 GiB OS + 0.8 GiB container are already in host usage.
|
|
// Adding a new 1 GiB server fits below 2.53 GiB usable memory.
|
|
heartbeat(hostId, {
|
|
...host,
|
|
memoryAvailable: 2 * GiB,
|
|
instanceResources: [
|
|
{
|
|
instanceId: instance.id,
|
|
generation: instance.generation,
|
|
uptimeSeconds: 60,
|
|
cpuCores: 0.1,
|
|
memoryBytes: Math.floor(0.8 * GiB)
|
|
}
|
|
]
|
|
});
|
|
const second = queueNativeCreation({ ...input, plutoKey: 'second-key' });
|
|
expect(second).toMatchObject({ ok: true });
|
|
await advance();
|
|
expect(await operationFor(second.ok ? second.deploymentId : 'missing')).not.toBeNull();
|
|
});
|
|
it.each(['missing', 'old-generation'] as const)(
|
|
'keeps full reservations for a submitted launch with %s measurements',
|
|
async (evidence) => {
|
|
await prepareLibrary();
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const GiB = 1024 ** 3;
|
|
heartbeat(hostId, {
|
|
...inventory(),
|
|
memoryTotal: 3 * GiB,
|
|
memoryAvailable: Math.floor(2.8 * GiB)
|
|
});
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true);
|
|
if (!first.ok) return;
|
|
await advance();
|
|
const instance = nativeInstance(first.name)!;
|
|
heartbeat(hostId, {
|
|
...inventory(),
|
|
memoryTotal: 3 * GiB,
|
|
memoryAvailable: 2 * GiB,
|
|
instanceResources:
|
|
evidence === 'missing'
|
|
? []
|
|
: [
|
|
{
|
|
instanceId: instance.id,
|
|
generation: instance.generation + 1,
|
|
uptimeSeconds: 60,
|
|
cpuCores: 0.1,
|
|
memoryBytes: Math.floor(0.8 * GiB)
|
|
}
|
|
]
|
|
});
|
|
const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
|
|
expect(next.ok).toBe(true);
|
|
if (!next.ok) throw new Error(next.message);
|
|
await advance();
|
|
expect(await operationFor(next.deploymentId)).toBeNull();
|
|
expect((await detail(next.deploymentId)).stage).toContain(
|
|
'3.00 GiB projected, 2.53 GiB usable'
|
|
);
|
|
}
|
|
);
|
|
it('waits when capacity disappears after admission without sending a create command', async () => {
|
|
await prepareLibrary();
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true);
|
|
if (!first.ok) return;
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
|
|
await advance();
|
|
expect(await operationFor(first.deploymentId)).toBeNull();
|
|
expect((await detail(first.deploymentId)).stage).toContain('capacity');
|
|
});
|
|
it('uses recent history at placement while keeping admission to current inventory', async () => {
|
|
await prepareLibrary();
|
|
const other = await secondHost({ cpuUsagePercent: 30 });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available)
|
|
VALUES(?,?,16,95,?,?)`
|
|
)
|
|
.run(
|
|
hostId,
|
|
new Date(Date.now() - 60_000).toISOString(),
|
|
64 * 1024 ** 3,
|
|
60 * 1024 ** 3
|
|
);
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true);
|
|
if (!first.ok) return;
|
|
await advance();
|
|
expect((await operationFor(first.deploymentId))?.hostId).toBe(other.id);
|
|
});
|
|
it('persists nullable CPU/memory samples and prunes old history per host', async () => {
|
|
await prepareLibrary();
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,'2000-01-01')"
|
|
)
|
|
.run(hostId);
|
|
heartbeat(hostId, { ...inventory(), cpuUsagePercent: undefined });
|
|
closeDb();
|
|
const rows = db()
|
|
.prepare('SELECT * FROM ec_host_resource_samples WHERE host_id=? ORDER BY sampled_at')
|
|
.all(hostId) as {
|
|
cpu_usage_percent: number | null;
|
|
memory_available: number;
|
|
sampled_at: string;
|
|
}[];
|
|
expect(rows.at(-1)).toMatchObject({
|
|
cpu_usage_percent: null,
|
|
memory_available: 60 * 1024 ** 3
|
|
});
|
|
expect(rows.some((row) => row.sampled_at === '2000-01-01')).toBe(false);
|
|
});
|
|
it('checks both port protocols and skips draining hosts', async () => {
|
|
await prepareLibrary();
|
|
const other = await secondHost();
|
|
const { reserveHostPort, updateHost } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
for (let port = 28960; port <= 28970; port++)
|
|
reserveHostPort(hostId, 'external-' + port, { port, protocol: 'tcp' });
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const result = queueNativeCreation(input);
|
|
expect(result.ok).toBe(true);
|
|
if (!result.ok) return;
|
|
expect(nativeInstance(result.name)).toBeNull();
|
|
await advance();
|
|
expect(nativeInstance(result.name)?.host_id).toBe(other.id);
|
|
updateHost(other.id, { mode: 'draining' });
|
|
const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
|
|
expect(next.ok).toBe(true);
|
|
if (!next.ok) throw new Error(next.message);
|
|
await advance();
|
|
expect(await operationFor(next.deploymentId)).toBeNull();
|
|
});
|
|
it('spreads collection dispatch across hosts, allowing the same port on different hosts', async () => {
|
|
await prepareLibrary();
|
|
await secondHost({ cpuCount: 1, cpuUsagePercent: 0 });
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection } =
|
|
await import('../../src/lib/server/fleet/collections.js');
|
|
const blueprint = saveBlueprint({
|
|
name: 'Two lobbies',
|
|
entries: [{ game: 't5', mapSlug: 'theater', count: 2 }]
|
|
});
|
|
const preview = previewCollection(blueprint.id);
|
|
expect(preview.ready).toBe(true);
|
|
expect(preview.servers.map((server) => server.port)).toEqual([null, null]);
|
|
expect(
|
|
queueCollection({
|
|
previewId: preview.id,
|
|
keys: Object.fromEntries(
|
|
preview.servers.map((server) => [server.name!, 'key-' + server.name])
|
|
)
|
|
}).ok
|
|
).toBe(true);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
await advance();
|
|
// Concurrent ranking can select the same host before the first
|
|
// reservation commits. The held job retries against the new usage.
|
|
await advance();
|
|
expect(db().prepare('SELECT DISTINCT host_id FROM native_instances').all()).toHaveLength(2);
|
|
expect(db().prepare('SELECT port FROM native_instances').all()).toEqual([
|
|
{ port: 28960 },
|
|
{ port: 28960 }
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe('native verification resume and explicit failed-creation retry', () => {
|
|
async function failedCreation(state: 'failed' | 'interrupted' = 'failed') {
|
|
const job = await queue();
|
|
await advance();
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, original.id, {
|
|
state: 'failed',
|
|
error: 'Host runtime preflight failed'
|
|
});
|
|
await advance();
|
|
if (state === 'interrupted') {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?")
|
|
.run(job.deploymentId);
|
|
}
|
|
return { job, original };
|
|
}
|
|
|
|
it.each(['failed', 'interrupted'] as const)(
|
|
'retries a %s creation through a new rebuild while retaining the original failure',
|
|
async (state) => {
|
|
const { job, original } = await failedCreation(state);
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
expect(history.getDeployment(job.deploymentId)).toMatchObject({
|
|
canResumeVerification: false,
|
|
canRetryWithRebuild: true
|
|
});
|
|
const { POST: resume } =
|
|
await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
|
|
expect(resume({ params: { id: job.deploymentId } }).status).toBe(409);
|
|
const { POST: retry } =
|
|
await import('../../src/routes/api/provisioning/[id]/retry-rebuild/+server.js');
|
|
const response = await retry({ params: { id: job.deploymentId } });
|
|
expect(response.status).toBe(202);
|
|
const next = (await response.json()) as { ok: boolean; deploymentId: string };
|
|
expect(next.ok).toBe(true);
|
|
expect(next.deploymentId).not.toBe(job.deploymentId);
|
|
expect(await detail(next.deploymentId)).toMatchObject({
|
|
action: 'rebuild',
|
|
state: 'waiting',
|
|
server: job.name
|
|
});
|
|
expect(history.getDeployment(job.deploymentId)).toMatchObject({
|
|
canRetryWithRebuild: false,
|
|
deployment: { state, error: 'Host runtime preflight failed' },
|
|
progress: { recovery: { state: 'transferred', operationId: next.deploymentId } }
|
|
});
|
|
expect((await retry({ params: { id: job.deploymentId } })).status).toBe(409);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db()
|
|
.prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server=?')
|
|
.get(job.name)
|
|
).toEqual({ count: 2 });
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)).toMatchObject({
|
|
id: original.instanceId,
|
|
generation: 1,
|
|
port: original.payload.port
|
|
});
|
|
remote.state = 'missing';
|
|
await advance();
|
|
const rebuilding = (await operationFor(next.deploymentId))!;
|
|
expect(rebuilding).toMatchObject({
|
|
kind: 'instance.rebuild',
|
|
generation: 1,
|
|
instanceId: original.instanceId,
|
|
payload: {
|
|
name: job.name,
|
|
port: original.payload.port,
|
|
nextGeneration: 2,
|
|
env: { PLUTO_SERVER_KEY: 'saved-key' }
|
|
}
|
|
});
|
|
expect(rebuilding.id).not.toBe(original.id);
|
|
expect(nativeInstance(job.name)?.generation).toBe(1);
|
|
remote.state = 'running';
|
|
await acknowledge(next.deploymentId);
|
|
expect((await detail(next.deploymentId)).state).toBe('done');
|
|
expect(nativeInstance(job.name)?.generation).toBe(2);
|
|
// An interrupted original is reconciled to its already-failed receipt
|
|
// on the next worker pass; the failure itself is never cleared.
|
|
expect(await detail(job.deploymentId)).toMatchObject({
|
|
state: 'failed',
|
|
error: 'Host runtime preflight failed'
|
|
});
|
|
expect(await operationFor(job.deploymentId)).toMatchObject({
|
|
id: original.id,
|
|
state: 'failed'
|
|
});
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.rebuild','instance.restart') ORDER BY rowid"
|
|
)
|
|
.all()
|
|
).toEqual([{ kind: 'instance.create' }, { kind: 'instance.rebuild' }]);
|
|
}
|
|
);
|
|
|
|
it.each([
|
|
['create', false],
|
|
['create', true],
|
|
['rebuild', false],
|
|
['rebuild', true],
|
|
['recovery-restart', false],
|
|
['recovery-restart', true],
|
|
['recovery-rebuild', false],
|
|
['recovery-rebuild', true]
|
|
] as const)(
|
|
'resumes completed %s verification with the same command (receipt applied=%s)',
|
|
async (action, applied) => {
|
|
let job = await queue();
|
|
await advance();
|
|
if (action === 'rebuild') {
|
|
await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test');
|
|
if (!rebuild.ok) throw new Error(rebuild.message);
|
|
job = rebuild;
|
|
await advance();
|
|
} else if (action.startsWith('recovery')) {
|
|
remote.state = 'unhealthy';
|
|
await acknowledge(job.deploymentId);
|
|
if (action === 'recovery-rebuild') await acknowledge(job.deploymentId);
|
|
}
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
remote.state = 'starting';
|
|
reportHostOperation(hostId, operation.id, { state: 'completed' });
|
|
if (applied) await advance();
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
history.finishDeployment(job.deploymentId, {
|
|
ok: false,
|
|
message: 'Verification interrupted for inspection'
|
|
});
|
|
expect(history.getDeployment(job.deploymentId)).toMatchObject({
|
|
canResumeVerification: true,
|
|
canRetryWithRebuild: false
|
|
});
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const commands = db()
|
|
.prepare(
|
|
"SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid"
|
|
)
|
|
.all();
|
|
const { POST } =
|
|
await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
|
|
expect(POST({ params: { id: job.deploymentId } }).status).toBe(202);
|
|
remote.state = 'running';
|
|
await advance();
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid"
|
|
)
|
|
.all()
|
|
).toEqual(commands);
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.generation).toBe(
|
|
operation.generation! + (operation.kind === 'instance.rebuild' ? 1 : 0)
|
|
);
|
|
expect(history.getDeployment(job.deploymentId)!.events).toContainEqual(
|
|
expect.objectContaining({
|
|
kind: 'outcome',
|
|
message: 'Failed: Verification interrupted for inspection'
|
|
})
|
|
);
|
|
}
|
|
);
|
|
|
|
it.each(['pending', 'accepted', 'running', 'unknown', 'failed'] as const)(
|
|
'does not offer verification of a %s command or replay an ambiguous result',
|
|
async (state) => {
|
|
const job = await queue();
|
|
await advance();
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
if (state !== 'pending') reportHostOperation(hostId, operation.id, { state });
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
history.finishDeployment(job.deploymentId, {
|
|
ok: false,
|
|
message: 'Controller observation interrupted'
|
|
});
|
|
expect(history.getDeployment(job.deploymentId)).toMatchObject({
|
|
canResumeVerification: false,
|
|
canRetryWithRebuild: state === 'failed'
|
|
});
|
|
expect(history.resumeVerification(job.deploymentId)).toBe(false);
|
|
if (state !== 'failed') {
|
|
const { requestRetryWithRebuild } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({
|
|
ok: false,
|
|
status: 409
|
|
});
|
|
}
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
if (state === 'unknown')
|
|
db()
|
|
.prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?")
|
|
.run(job.deploymentId);
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')"
|
|
)
|
|
.all()
|
|
).toEqual([{ id: operation.id }]);
|
|
}
|
|
);
|
|
|
|
it.each([
|
|
'instance generation',
|
|
'command generation',
|
|
'command instance',
|
|
'command host',
|
|
'retired instance',
|
|
'server port',
|
|
'newer work',
|
|
'creation cleanup',
|
|
'recovery attempt'
|
|
])('refuses to retry a failed creation after its %s changes', async (change) => {
|
|
const { job, original } = await failedCreation();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
if (change === 'instance generation')
|
|
db()
|
|
.prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?')
|
|
.run(job.name);
|
|
if (change === 'command generation')
|
|
db()
|
|
.prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?')
|
|
.run(original.id);
|
|
if (change === 'command instance')
|
|
db()
|
|
.prepare(
|
|
"UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?"
|
|
)
|
|
.run(original.id);
|
|
if (change === 'command host') {
|
|
const { createHost } = await import('../../src/lib/server/hosts/index.js');
|
|
db()
|
|
.prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?')
|
|
.run(createHost({ name: 'Different host' }).id, original.id);
|
|
}
|
|
if (change === 'retired instance')
|
|
db()
|
|
.prepare('UPDATE native_instances SET retired_at=? WHERE server=?')
|
|
.run(new Date().toISOString(), job.name);
|
|
if (change === 'server port')
|
|
db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
|
|
if (change === 'newer work') {
|
|
const { beginDeployment } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
beginDeployment(job.name, 'restart');
|
|
}
|
|
if (change === 'creation cleanup') {
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
|
|
}
|
|
if (change === 'recovery attempt')
|
|
db()
|
|
.prepare(
|
|
"UPDATE native_provisioning SET recovery_step='restart' WHERE deployment_id=?"
|
|
)
|
|
.run(job.deploymentId);
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
const { requestRetryWithRebuild } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
expect(history.getDeployment(job.deploymentId)?.canRetryWithRebuild).toBe(false);
|
|
expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 });
|
|
expect(
|
|
db().prepare("SELECT id FROM server_deployments WHERE action='rebuild'").all()
|
|
).toEqual([]);
|
|
});
|
|
|
|
it.each([
|
|
'instance generation',
|
|
'command generation',
|
|
'command instance',
|
|
'command host',
|
|
'retired instance',
|
|
'server port',
|
|
'newer work'
|
|
])('refuses to resume a completed creation after its %s changes', async (change) => {
|
|
const job = await queue();
|
|
await advance();
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation, createHost } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, original.id, { state: 'completed' });
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
history.finishDeployment(job.deploymentId, {
|
|
ok: false,
|
|
message: 'Verification interrupted'
|
|
});
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
if (change === 'instance generation')
|
|
db()
|
|
.prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?')
|
|
.run(job.name);
|
|
if (change === 'command generation')
|
|
db()
|
|
.prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?')
|
|
.run(original.id);
|
|
if (change === 'command instance')
|
|
db()
|
|
.prepare(
|
|
"UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?"
|
|
)
|
|
.run(original.id);
|
|
if (change === 'command host')
|
|
db()
|
|
.prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?')
|
|
.run(createHost({ name: 'Different host' }).id, original.id);
|
|
if (change === 'retired instance')
|
|
db()
|
|
.prepare('UPDATE native_instances SET retired_at=? WHERE server=?')
|
|
.run(new Date().toISOString(), job.name);
|
|
if (change === 'server port')
|
|
db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
|
|
if (change === 'newer work') history.beginDeployment(job.name, 'restart');
|
|
expect(history.getDeployment(job.deploymentId)?.canResumeVerification).toBe(false);
|
|
expect(history.resumeVerification(job.deploymentId)).toBe(false);
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
|
});
|
|
});
|
|
|
|
describe('unhealthy recovery within native create and rebuild operations', () => {
|
|
async function launch(action: 'create' | 'rebuild') {
|
|
const job = await queue();
|
|
await advance();
|
|
if (action === 'create') return job;
|
|
await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test');
|
|
if (!rebuild.ok) throw new Error(rebuild.message);
|
|
await advance();
|
|
return rebuild;
|
|
}
|
|
|
|
it.each(['create', 'rebuild'] as const)(
|
|
'recovers an unhealthy %s with one restart then a rebuild using the original launch payload',
|
|
async (action) => {
|
|
const { config } = await import('../../src/lib/server/config.js');
|
|
Object.assign(config.rcon, { password: '' });
|
|
const job = await launch(action);
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const currentGeneration = action === 'rebuild' ? 2 : 1;
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconStatusResult).mockClear();
|
|
remote.state = 'unhealthy';
|
|
remote.players = 4;
|
|
Object.assign(config.rcon, { password: 'changed-after-original-launch' });
|
|
await acknowledge(job.deploymentId);
|
|
const restart = (await operationFor(job.deploymentId))!;
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
expect(restart).toMatchObject({
|
|
kind: 'instance.restart',
|
|
state: 'pending',
|
|
generation: currentGeneration,
|
|
instanceId: original.instanceId,
|
|
payload: { unhealthyRecovery: true }
|
|
});
|
|
expect(getDeployment(job.deploymentId)!.capacity.pools).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ kind: 'startup', ownsSlot: true }),
|
|
expect.objectContaining({ kind: 'restart', ownsSlot: true })
|
|
])
|
|
);
|
|
expect(restart.id).not.toBe(original.id);
|
|
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?'
|
|
)
|
|
.get(job.deploymentId)
|
|
).toEqual({ recovery_step: 'restart', generation: currentGeneration });
|
|
expect(rconStatusResult).not.toHaveBeenCalled();
|
|
|
|
// Restarting the controller must preserve the current recovery command.
|
|
closeDb();
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(restart.id);
|
|
await acknowledge(job.deploymentId);
|
|
const rebuild = (await operationFor(job.deploymentId))!;
|
|
expect(rebuild).toMatchObject({
|
|
kind: 'instance.rebuild',
|
|
state: 'pending',
|
|
generation: currentGeneration,
|
|
instanceId: original.instanceId,
|
|
payload: {
|
|
unhealthyRecovery: true,
|
|
nextGeneration: currentGeneration + 1,
|
|
files: original.payload.files,
|
|
env: original.payload.env,
|
|
rconPassword: original.payload.rconPassword,
|
|
libraryId: original.payload.libraryId,
|
|
image: original.payload.image
|
|
}
|
|
});
|
|
expect(rebuild.id).not.toBe(restart.id);
|
|
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration);
|
|
expect(rconStatusResult).not.toHaveBeenCalled();
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?'
|
|
)
|
|
.get(job.deploymentId)
|
|
).toEqual({ recovery_step: 'rebuild', generation: currentGeneration });
|
|
remote.state = 'running';
|
|
remote.players = 0;
|
|
await acknowledge(job.deploymentId);
|
|
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1);
|
|
expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' });
|
|
expect(getDeployment(job.deploymentId)!.progress?.recovery?.state).toBe('released');
|
|
expect(
|
|
getDeployment(job.deploymentId)!.capacity.pools.every((pool) => !pool.ownsSlot)
|
|
).toBe(true);
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
|
|
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1);
|
|
}
|
|
);
|
|
|
|
it.each(['create', 'rebuild'] as const)(
|
|
'finishes %s after a successful recovery restart without another rebuild',
|
|
async (action) => {
|
|
const job = await launch(action);
|
|
remote.state = 'unhealthy';
|
|
await acknowledge(job.deploymentId);
|
|
const restart = (await operationFor(job.deploymentId))!;
|
|
expect(restart.kind).toBe('instance.restart');
|
|
remote.state = 'running';
|
|
await acknowledge(job.deploymentId);
|
|
await advance();
|
|
expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' });
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(restart.id);
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 2 : 1);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'"
|
|
)
|
|
.get()
|
|
).toEqual({ count: action === 'rebuild' ? 1 : 0 });
|
|
}
|
|
);
|
|
|
|
it.each(['restart', 'rebuild'] as const)(
|
|
'retains an unknown recovery %s receipt without replaying or advancing recovery',
|
|
async (step) => {
|
|
const job = await launch('create');
|
|
remote.state = 'unhealthy';
|
|
await acknowledge(job.deploymentId);
|
|
if (step === 'rebuild') await acknowledge(job.deploymentId);
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
expect(operation.kind).toBe(`instance.${step}`);
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'unknown',
|
|
error: 'Reply lost during recovery'
|
|
});
|
|
const { closeDb, db } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
const { interruptDeployments } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
interruptDeployments();
|
|
await advance();
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')"
|
|
)
|
|
.get()
|
|
).toEqual({ count: step === 'restart' ? 2 : 3 });
|
|
expect((await detail(job.deploymentId)).state).not.toBe('failed');
|
|
remote.state = 'running';
|
|
await acknowledge(job.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
}
|
|
);
|
|
|
|
it.each(['starting', 'restarting', 'unknown'])(
|
|
'waits for %s container readiness without forcing recovery',
|
|
async (state) => {
|
|
const job = await launch('create');
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
remote.state = state;
|
|
await acknowledge(job.deploymentId);
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
|
expect(await detail(job.deploymentId)).toMatchObject({
|
|
state: 'awaiting-health',
|
|
stage: `Waiting for container readiness: ${state}`
|
|
});
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db()
|
|
.prepare('SELECT recovery_step FROM native_provisioning WHERE deployment_id=?')
|
|
.get(job.deploymentId)
|
|
).toEqual({ recovery_step: null });
|
|
}
|
|
);
|
|
|
|
it.each(['create', 'rebuild'] as const)(
|
|
'does not submit recovery if %s is cancelled during its health probe',
|
|
async (action) => {
|
|
const job = await launch(action);
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, original.id, { state: 'completed' });
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(
|
|
() =>
|
|
new Promise((resolve) => {
|
|
release = resolve;
|
|
})
|
|
);
|
|
const pending = advance();
|
|
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
|
expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
|
|
release({ healthy: false, state: 'unhealthy' } as never);
|
|
await pending;
|
|
await advance();
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db()
|
|
.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'")
|
|
.all()
|
|
).toEqual([]);
|
|
}
|
|
);
|
|
|
|
it.each(['create', 'rebuild'] as const)(
|
|
'reconciles a delivered recovery rebuild after its %s flow is cancelled',
|
|
async (action) => {
|
|
const job = await launch(action);
|
|
remote.state = 'unhealthy';
|
|
await acknowledge(job.deploymentId);
|
|
await acknowledge(job.deploymentId);
|
|
const rebuild = (await operationFor(job.deploymentId))!;
|
|
expect(rebuild.kind).toBe('instance.rebuild');
|
|
const { pendingHostOperations, reportHostOperation } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
pendingHostOperations(hostId);
|
|
expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
|
|
reportHostOperation(hostId, rebuild.id, { state: 'completed' });
|
|
await advance();
|
|
await advance();
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 3 : 2);
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
|
|
}
|
|
);
|
|
|
|
it('fails clearly after an unhealthy recovery rebuild instead of looping', async () => {
|
|
const job = await launch('create');
|
|
remote.state = 'unhealthy';
|
|
await acknowledge(job.deploymentId);
|
|
await acknowledge(job.deploymentId);
|
|
const rebuild = (await operationFor(job.deploymentId))!;
|
|
expect(rebuild.kind).toBe('instance.rebuild');
|
|
await acknowledge(job.deploymentId);
|
|
await advance();
|
|
expect(await detail(job.deploymentId)).toMatchObject({
|
|
state: 'failed',
|
|
error: expect.stringMatching(/unhealthy.*restart.*rebuild/i)
|
|
});
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')"
|
|
)
|
|
.get()
|
|
).toEqual({ count: 3 });
|
|
});
|
|
|
|
it.each(['create', 'rebuild'] as const)(
|
|
'holds unhealthy %s verification until the agent supports recovery',
|
|
async (action) => {
|
|
const job = await launch(action);
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), capabilities: [] });
|
|
remote.state = 'unhealthy';
|
|
await acknowledge(job.deploymentId);
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
|
expect(await detail(job.deploymentId)).toMatchObject({
|
|
state: 'awaiting-health',
|
|
stage: expect.stringMatching(/agent/i)
|
|
});
|
|
heartbeat(hostId, inventory());
|
|
await advance();
|
|
expect(await operationFor(job.deploymentId)).toMatchObject({
|
|
kind: 'instance.restart',
|
|
payload: { unhealthyRecovery: true }
|
|
});
|
|
}
|
|
);
|
|
|
|
it.each([false, true])(
|
|
'rebuilds an already unhealthy container without requiring RCON emptiness once supported (supported=%s)',
|
|
async (supported) => {
|
|
const created = await launch('create');
|
|
await acknowledge(created.deploymentId);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
if (!supported) heartbeat(hostId, { ...inventory(), capabilities: [] });
|
|
const { queueNativeRebuild } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(created.name, 'test');
|
|
if (!rebuild.ok) throw new Error(rebuild.message);
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconStatusResult).mockClear();
|
|
remote.state = 'unhealthy';
|
|
remote.players = 3;
|
|
await advance();
|
|
if (!supported) {
|
|
expect(await operationFor(rebuild.deploymentId)).toBeNull();
|
|
expect((await detail(rebuild.deploymentId)).stage).toMatch(/agent/i);
|
|
heartbeat(hostId, inventory());
|
|
await advance();
|
|
}
|
|
expect(await operationFor(rebuild.deploymentId)).toMatchObject({
|
|
kind: 'instance.rebuild',
|
|
generation: 1,
|
|
payload: { unhealthyRecovery: true, nextGeneration: 2 }
|
|
});
|
|
expect(rconStatusResult).not.toHaveBeenCalled();
|
|
}
|
|
);
|
|
});
|
|
|
|
describe('native provisioning cancellation and rollback', () => {
|
|
it('cancels an unhealthy restart and prevents a late health response from reviving it', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const restart = requestLifecycle(job.name, 'restart');
|
|
if (!restart.ok) throw new Error(restart.error);
|
|
await advance();
|
|
const command = (await operationFor(restart.deploymentId))!;
|
|
reportHostOperation(hostId, command.id, { state: 'completed' });
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(
|
|
() =>
|
|
new Promise((resolve) => {
|
|
release = resolve;
|
|
})
|
|
);
|
|
const pending = advance();
|
|
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
|
expect(POST({ params: { id: restart.deploymentId } }).status).toBe(200);
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await pending;
|
|
expect(await detail(restart.deploymentId)).toMatchObject({ state: 'cancelled' });
|
|
const replacement = requestLifecycle(job.name, 'restart');
|
|
if (!replacement.ok) throw new Error(replacement.error);
|
|
await advance();
|
|
expect(await operationFor(replacement.deploymentId)).toMatchObject({
|
|
kind: 'instance.restart',
|
|
state: 'pending'
|
|
});
|
|
});
|
|
|
|
it.each([false, true])(
|
|
'withdraws an undelivered restart or waits for a delivered one before replacement (delivered=%s)',
|
|
async (delivered) => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { requestLifecycle } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
const { pendingHostOperations, reportHostOperation } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const first = requestLifecycle(job.name, 'restart');
|
|
if (!first.ok) throw new Error(first.error);
|
|
await advance();
|
|
const command = (await operationFor(first.deploymentId))!;
|
|
if (delivered) pendingHostOperations(hostId);
|
|
expect(cancelProvisioningJob(first.deploymentId)).toBe(true);
|
|
const next = requestLifecycle(job.name, 'restart');
|
|
if (!next.ok) throw new Error(next.error);
|
|
await advance();
|
|
if (delivered) {
|
|
expect(await operationFor(next.deploymentId)).toBeNull();
|
|
reportHostOperation(hostId, command.id, { state: 'completed' });
|
|
await advance();
|
|
} else {
|
|
expect((await operationFor(first.deploymentId))?.state).toBe('failed');
|
|
expect(pendingHostOperations(hostId).some((op) => op.id === command.id)).toBe(
|
|
false
|
|
);
|
|
}
|
|
expect(await operationFor(next.deploymentId)).not.toBeNull();
|
|
expect((await detail(first.deploymentId)).state).toBe('cancelled');
|
|
}
|
|
);
|
|
|
|
it('rolls back a completed native activation even while its restart is stuck starting', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { script, version } = createScript({
|
|
releaseVersion: '1.0.0',
|
|
name: 'Broken beta',
|
|
kind: 'script',
|
|
targetRelPath: 'beta.gsc',
|
|
content: Buffer.from('broken'),
|
|
scopeType: 'game',
|
|
scopeValue: 't5'
|
|
});
|
|
const betaId = Number(
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')"
|
|
)
|
|
.run(script.id, version.id).lastInsertRowid
|
|
);
|
|
db()
|
|
.prepare('INSERT INTO beta_scripts(beta_id,script_id,script_version_id) VALUES(?,?,?)')
|
|
.run(betaId, script.id, version.id);
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')"
|
|
)
|
|
.run(betaId, job.name);
|
|
db()
|
|
.prepare(
|
|
'INSERT INTO beta_server_scripts(beta_id,server,script_id,had_original) VALUES(?,?,?,0)'
|
|
)
|
|
.run(betaId, job.name, script.id);
|
|
queueFileChange({
|
|
sourceKind: 'beta',
|
|
sourceId: String(betaId),
|
|
server: job.name,
|
|
kind: 'script',
|
|
relPath: 'beta.gsc',
|
|
content: Buffer.from('broken'),
|
|
inventorySource: 'beta'
|
|
});
|
|
const restart = requestRestart(job.name, { kind: 'beta', id: String(betaId) });
|
|
await advance();
|
|
const staging = db()
|
|
.prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
|
|
.get(restart.id) as { stage_operation_id: string };
|
|
reportHostOperation(hostId, staging.stage_operation_id, { state: 'completed' });
|
|
await advance();
|
|
const activation = (await operationFor(restart.id))!;
|
|
const files = JSON.parse(
|
|
(
|
|
db()
|
|
.prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?')
|
|
.get(restart.id) as { files_json: string }
|
|
).files_json
|
|
);
|
|
remote.state = 'starting';
|
|
reportHostOperation(hostId, activation.id, {
|
|
state: 'completed',
|
|
result: {
|
|
before: files.map((f: { id: number; path: string }) => ({
|
|
...f,
|
|
contentBase64: Buffer.from('working').toString('base64')
|
|
}))
|
|
}
|
|
});
|
|
// Rollback must consume the receipt itself; the health worker has not caught up.
|
|
const rollback = await rollbackBeta(betaId);
|
|
expect((await detail(restart.id)).state).toBe('cancelled');
|
|
const replacement = db()
|
|
.prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id=?')
|
|
.get(rollback.deploymentId) as { provisioning_id: string };
|
|
expect(replacement.provisioning_id).not.toBe(restart.id);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?"
|
|
)
|
|
.get(rollback.deploymentId)
|
|
).toEqual({ content: Buffer.from('working') });
|
|
await advance();
|
|
const stagedRollback = db()
|
|
.prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
|
|
.get(replacement.provisioning_id) as { stage_operation_id: string };
|
|
reportHostOperation(hostId, stagedRollback.stage_operation_id, { state: 'completed' });
|
|
await advance();
|
|
expect(await operationFor(replacement.provisioning_id)).toMatchObject({ state: 'pending' });
|
|
});
|
|
});
|
|
|
|
describe('partial rebuild failure reconciliation', () => {
|
|
const originalError = 'Replacement game library could not be prepared';
|
|
const failure = (generation = 1) => ({
|
|
reconciled: true,
|
|
originalError,
|
|
rebuildFailure: {
|
|
phase: 'disrupting',
|
|
generation,
|
|
nextGeneration: generation + 1,
|
|
oldContainerMissing: true,
|
|
replacementContainerMissing: true,
|
|
replacementFilesRemoved: true
|
|
}
|
|
});
|
|
|
|
async function cancelledRebuild() {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const first = requestLifecycle(job.name, 'rebuild');
|
|
if (!first.ok) throw new Error(first.error);
|
|
await advance();
|
|
const command = (await operationFor(first.deploymentId))!;
|
|
const { pendingHostOperations, reportHostOperation } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
pendingHostOperations(hostId);
|
|
reportHostOperation(hostId, command.id, { state: 'unknown', error: originalError });
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
expect(cancelProvisioningJob(first.deploymentId)).toBe(true);
|
|
const replacement = requestLifecycle(job.name, 'rebuild');
|
|
if (!replacement.ok) throw new Error(replacement.error);
|
|
return { job, first, command, replacement };
|
|
}
|
|
|
|
it('records missing containers before releasing a queued rebuild without replaying the cancelled command', async () => {
|
|
const { job, first, command, replacement } = await cancelledRebuild();
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { getHostOperation, reportHostOperation, pendingHostOperations } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const { nativeInstance, nativeRequest } =
|
|
await import('../../src/lib/server/backends/native.js');
|
|
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { unsettledCancelledCommand } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
const ports = db()
|
|
.prepare(
|
|
'SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol'
|
|
)
|
|
.all();
|
|
await advance();
|
|
expect(await operationFor(replacement.deploymentId)).toBeNull();
|
|
expect(getDeployment(replacement.deploymentId)?.progress?.blocker).toMatchObject({
|
|
code: 'cancelled-command',
|
|
href: `/provisioning/${first.deploymentId}`,
|
|
label: 'View cancelled operation',
|
|
message: expect.stringContaining(originalError)
|
|
});
|
|
expect(getDeployment(replacement.deploymentId)?.progress?.blocker?.message).toContain(
|
|
'unknown'
|
|
);
|
|
reportHostOperation(hostId, command.id, {
|
|
state: 'failed',
|
|
error: `Old container removed; replacement creation failed. ${originalError}`,
|
|
result: failure()
|
|
});
|
|
expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).not.toBeNull();
|
|
expect(
|
|
db()
|
|
.prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?')
|
|
.get(first.deploymentId)
|
|
).toEqual({ applied_at: null });
|
|
remote.state = 'missing';
|
|
vi.mocked(nativeRequest).mockImplementationOnce(async () => {
|
|
expect(
|
|
db()
|
|
.prepare('SELECT container_state,last_players FROM servers WHERE name=?')
|
|
.get(job.name)
|
|
).toEqual({ container_state: 'missing', last_players: null });
|
|
expect(
|
|
db()
|
|
.prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?')
|
|
.get(first.deploymentId)
|
|
).toEqual({ applied_at: expect.any(String) });
|
|
expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).toBeNull();
|
|
return { exists: false, healthy: false, state: 'missing' } as never;
|
|
});
|
|
closeDb();
|
|
await advance();
|
|
const next = (await operationFor(replacement.deploymentId))!;
|
|
expect(next).toMatchObject({
|
|
kind: 'instance.rebuild',
|
|
generation: 1,
|
|
instanceId: command.instanceId,
|
|
payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } }
|
|
});
|
|
expect(next.id).not.toBe(command.id);
|
|
expect(nativeInstance(job.name)).toMatchObject({
|
|
id: command.instanceId,
|
|
generation: 1,
|
|
retired_at: null
|
|
});
|
|
expect(getServerKey(job.name)).toBe('saved-key');
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol'
|
|
)
|
|
.all()
|
|
).toEqual(ports);
|
|
expect((await detail(first.deploymentId)).state).toBe('cancelled');
|
|
expect(getHostOperation(command.id)).toMatchObject({ state: 'failed', result: failure() });
|
|
expect(pendingHostOperations(hostId).some((operation) => operation.id === command.id)).toBe(
|
|
false
|
|
);
|
|
await advance();
|
|
expect((await operationFor(replacement.deploymentId))!.id).toBe(next.id);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'"
|
|
)
|
|
.get()
|
|
).toEqual({ count: 2 });
|
|
remote.state = 'running';
|
|
await acknowledge(replacement.deploymentId);
|
|
expect((await detail(replacement.deploymentId)).state).toBe('done');
|
|
expect(nativeInstance(job.name)?.generation).toBe(2);
|
|
expect((await detail(first.deploymentId)).state).toBe('cancelled');
|
|
});
|
|
|
|
it.each(['disrupting', 'creating'])(
|
|
'keeps an ambiguous %s rebuild blocked even when inspections currently see no containers',
|
|
async (phase) => {
|
|
const { job, first, command, replacement } = await cancelledRebuild();
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, command.id, {
|
|
state: 'unknown',
|
|
error: originalError,
|
|
result: {
|
|
checkpoint: { phase },
|
|
state: 'missing',
|
|
oldContainerMissing: true,
|
|
replacementContainerMissing: true
|
|
}
|
|
});
|
|
remote.state = 'missing';
|
|
const { nativeRequest, nativeInstance } =
|
|
await import('../../src/lib/server/backends/native.js');
|
|
vi.mocked(nativeRequest).mockClear();
|
|
const { closeDb, db } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
await advance();
|
|
await advance();
|
|
expect(await operationFor(replacement.deploymentId)).toBeNull();
|
|
expect(vi.mocked(nativeRequest)).not.toHaveBeenCalled();
|
|
expect((await operationFor(first.deploymentId))!.id).toBe(command.id);
|
|
expect(nativeInstance(job.name)?.generation).toBe(1);
|
|
expect(
|
|
db()
|
|
.prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?')
|
|
.get(first.deploymentId)
|
|
).toEqual({ applied_at: null });
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'"
|
|
)
|
|
.get()
|
|
).toEqual({ count: 1 });
|
|
}
|
|
);
|
|
|
|
it.each([
|
|
'creation started',
|
|
'replacement files remain',
|
|
'wrong target generation',
|
|
'command target mismatch'
|
|
])('does not release a replacement from invalid partial evidence: %s', async (evidence) => {
|
|
const { first, command, replacement } = await cancelledRebuild();
|
|
const result = failure();
|
|
if (evidence === 'creation started') result.rebuildFailure.phase = 'creating';
|
|
if (evidence === 'replacement files remain')
|
|
result.rebuildFailure.replacementFilesRemoved = false;
|
|
if (evidence === 'wrong target generation') result.rebuildFailure.nextGeneration = 3;
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
if (evidence === 'command target mismatch')
|
|
db()
|
|
.prepare(
|
|
"UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?"
|
|
)
|
|
.run(command.id);
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, command.id, { state: 'failed', error: originalError, result });
|
|
remote.state = 'missing';
|
|
await advance();
|
|
expect(await operationFor(replacement.deploymentId)).toBeNull();
|
|
expect(
|
|
db()
|
|
.prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?')
|
|
.get(first.deploymentId)
|
|
).toEqual({ applied_at: null });
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'"
|
|
)
|
|
.get()
|
|
).toEqual({ count: 1 });
|
|
});
|
|
|
|
it('records an uncancelled partial rebuild failure without claiming replacement generation or successful health', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const first = requestLifecycle(job.name, 'rebuild');
|
|
if (!first.ok) throw new Error(first.error);
|
|
await advance();
|
|
const command = (await operationFor(first.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, command.id, {
|
|
state: 'failed',
|
|
result: failure(),
|
|
error: `Old container removed; replacement creation failed. ${originalError}`
|
|
});
|
|
await advance();
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect((await detail(first.deploymentId)).state).toBe('failed');
|
|
expect(nativeInstance(job.name)).toMatchObject({ generation: 1, retired_at: null });
|
|
expect(
|
|
db()
|
|
.prepare('SELECT container_state,last_players FROM servers WHERE name=?')
|
|
.get(job.name)
|
|
).toEqual({ container_state: 'missing', last_players: null });
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT applied_at,verified_at FROM native_provisioning WHERE deployment_id=?'
|
|
)
|
|
.get(first.deploymentId)
|
|
).toEqual({ applied_at: expect.any(String), verified_at: null });
|
|
expect(
|
|
db()
|
|
.prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE instance_id=?')
|
|
.get(command.instanceId)
|
|
).toEqual({ count: 2 });
|
|
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
expect(getServerKey(job.name)).toBe('saved-key');
|
|
});
|
|
});
|
|
|
|
describe('cancelled native mutation bookkeeping', () => {
|
|
it.each([null, 'before beta'])(
|
|
'keeps the original host receipt across beta revisions (original=%s)',
|
|
async (original) => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta, reviseBeta, rollbackBeta } =
|
|
await import('../../src/lib/server/fleet/beta.js');
|
|
const { requestRestart } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
const { reportHostOperation, getHostOperation } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const { script, version } = createScript({
|
|
releaseVersion: '1.0.0',
|
|
name: 'Revised beta',
|
|
kind: 'script',
|
|
targetRelPath: 'revision.gsc',
|
|
content: Buffer.from('revision one'),
|
|
scopeType: 'game',
|
|
scopeValue: 't5'
|
|
});
|
|
const { betaId } = await openBeta({
|
|
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
|
servers: [job.name],
|
|
waitForHealth: true,
|
|
waitForMatch: false
|
|
});
|
|
async function activate(before: string | null) {
|
|
const restart = requestRestart(job.name, { kind: 'beta', id: String(betaId) });
|
|
await advance();
|
|
const staged = db()
|
|
.prepare(
|
|
'SELECT stage_operation_id FROM native_provisioning WHERE deployment_id = ?'
|
|
)
|
|
.get(restart.id) as { stage_operation_id: string };
|
|
const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{
|
|
id: number;
|
|
path: string;
|
|
}>;
|
|
reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
|
|
await advance();
|
|
// A dispatched activation must settle before revision submission.
|
|
await expect(
|
|
reviseBeta(betaId, 1, [
|
|
{
|
|
scriptId: script.id,
|
|
releaseVersion: '1.0.1',
|
|
content: Buffer.from('premature update')
|
|
}
|
|
])
|
|
).rejects.toThrow();
|
|
const activation = (await operationFor(restart.id))!;
|
|
expect(activation.kind).toBe('instance.rebuild');
|
|
expect(activation.payload).toMatchObject({
|
|
stageId: restart.id,
|
|
nextGeneration: activation.generation! + 1
|
|
});
|
|
expect(activation.payload).not.toHaveProperty('unhealthyRecovery');
|
|
reportHostOperation(hostId, activation.id, {
|
|
state: 'completed',
|
|
result: {
|
|
before: files.map((file) => ({
|
|
...file,
|
|
contentBase64:
|
|
before === null ? null : Buffer.from(before).toString('base64')
|
|
}))
|
|
}
|
|
});
|
|
await advance();
|
|
expect((await detail(restart.id)).state).toBe('done');
|
|
}
|
|
await activate(original);
|
|
await reviseBeta(betaId, 1, [
|
|
{
|
|
scriptId: script.id,
|
|
releaseVersion: '1.0.1',
|
|
content: Buffer.from('revision two')
|
|
}
|
|
]);
|
|
await activate('revision one');
|
|
const rollback = await rollbackBeta(betaId);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?"
|
|
)
|
|
.get(rollback.deploymentId)
|
|
).toEqual({ content: original === null ? null : Buffer.from(original) });
|
|
}
|
|
);
|
|
|
|
it('keeps explicit cleanup available after cancelling an unsubmitted creation', async () => {
|
|
const job = await queue();
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
const { cancelCreation, canCancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
|
|
expect(canCancelCreation(job.deploymentId)).toBe(true);
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
expect((await detail(cleanup)).state).toBe('done');
|
|
});
|
|
|
|
it.each(['rebuild', 'destroy'] as const)(
|
|
'reconciles a delivered %s receipt without resuming the cancelled job',
|
|
async (action) => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { serverKeyPool, getServerKey } =
|
|
await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { requestLifecycle } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
const { destroyServer } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
const { pendingHostOperations, reportHostOperation } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const result =
|
|
action === 'rebuild'
|
|
? requestLifecycle(job.name, 'rebuild')
|
|
: destroyServer(job.name);
|
|
if (!result.ok || !result.deploymentId) throw new Error('Job was not queued');
|
|
const id = result.deploymentId;
|
|
await advance();
|
|
const operation = (await operationFor(id))!;
|
|
pendingHostOperations(hostId);
|
|
expect(cancelProvisioningJob(id)).toBe(true);
|
|
expect(serverKeyPool('t5').available).toBe(0);
|
|
const replacement = action === 'rebuild' ? requestLifecycle(job.name, 'restart') : null;
|
|
if (replacement && !replacement.ok) throw new Error(replacement.error);
|
|
await advance();
|
|
reportHostOperation(hostId, operation.id, { state: 'completed' });
|
|
await advance();
|
|
await advance();
|
|
if (replacement?.ok)
|
|
expect(await operationFor(replacement.deploymentId)).toMatchObject({
|
|
kind: 'instance.restart',
|
|
generation: 2
|
|
});
|
|
expect((await detail(id)).state).toBe('cancelled');
|
|
expect(serverKeyPool('t5').available).toBe(action === 'destroy' ? 1 : 0);
|
|
expect(getServerKey(job.name)).toBe(action === 'destroy' ? undefined : 'saved-key');
|
|
if (action === 'rebuild')
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT generation,retired_at FROM native_instances WHERE server=?'
|
|
)
|
|
.get(job.name)
|
|
).toEqual({ generation: 2, retired_at: null });
|
|
else {
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
expect(
|
|
db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)
|
|
).toBeUndefined();
|
|
}
|
|
}
|
|
);
|
|
|
|
it.each([false, true])(
|
|
'cancels a host migration without restarting it or leaking destination reservations (delivered=%s)',
|
|
async (delivered) => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const { serverKeyPool, getServerKey } =
|
|
await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { getLibraryVersion, updateHostLibraryCaches } =
|
|
await import('../../src/lib/server/game-files/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { processProvisioning } =
|
|
await import('../../src/lib/server/provisioning/worker.js');
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
const destination = hosts.createHost({
|
|
name: 'Destination',
|
|
portMin: 29000,
|
|
portMax: 29005
|
|
});
|
|
hosts.enrollHost(hosts.createEnrollment(destination.id).token, inventory());
|
|
updateHostLibraryCaches(destination.id, [
|
|
{
|
|
libraryVersionId: libraryId,
|
|
status: 'ready',
|
|
manifestSha256: getLibraryVersion(libraryId)!.manifestSha256
|
|
}
|
|
]);
|
|
hosts.updateHost(hostId, { mode: 'draining' });
|
|
await processProvisioning();
|
|
const migration = db()
|
|
.prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations')
|
|
.get() as { deployment_id: string; remove_operation_id: string };
|
|
expect(migration.remove_operation_id).toBeTruthy();
|
|
if (delivered) hosts.pendingHostOperations(hostId);
|
|
expect(cancelProvisioningJob(migration.deployment_id)).toBe(true);
|
|
expect(serverKeyPool('t5').available).toBe(0);
|
|
if (delivered)
|
|
hosts.reportHostOperation(hostId, migration.remove_operation_id, {
|
|
state: 'completed'
|
|
});
|
|
await processProvisioning();
|
|
await processProvisioning();
|
|
expect((await detail(migration.deployment_id)).state).toBe('cancelled');
|
|
expect(
|
|
db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(destination.id)
|
|
).toEqual([]);
|
|
expect(db().prepare('SELECT * FROM ec_host_migrations').all()).toHaveLength(1);
|
|
expect(
|
|
db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)
|
|
).toHaveLength(delivered ? 0 : 2);
|
|
expect(serverKeyPool('t5').available).toBe(delivered ? 1 : 0);
|
|
expect(getServerKey(job.name)).toBe(delivered ? undefined : 'saved-key');
|
|
}
|
|
);
|
|
});
|
|
|
|
describe('additional concurrency admission', () => {
|
|
it.each(['createConcurrencyPerHost', 'lifecycleConcurrency'] as const)(
|
|
'holds simultaneous startups with %s until recovery completes',
|
|
async (key) => {
|
|
const first = await queue();
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 5, [key]: 1 });
|
|
const { queueCreateServer } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
const second = queueCreateServer({
|
|
game: 't5',
|
|
gameMode: 'zombie',
|
|
mapSlug: 'theater',
|
|
hostId,
|
|
libraryId,
|
|
image,
|
|
plutoKey: 'second-key'
|
|
});
|
|
if (!second.ok) throw new Error(second.message);
|
|
await advance();
|
|
const operations = [
|
|
await operationFor(first.deploymentId),
|
|
await operationFor(second.deploymentId)
|
|
];
|
|
expect(operations.filter(Boolean)).toHaveLength(1);
|
|
expect((await detail(second.deploymentId)).stage).toContain('capacity');
|
|
saveFleetSettings({ ...fleetSettings(), [key]: 0 });
|
|
await advance();
|
|
expect(await operationFor(second.deploymentId)).not.toBeNull();
|
|
}
|
|
);
|
|
});
|
|
|
|
describe('match history on server stop', () => {
|
|
async function setupStop() {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const handle = db();
|
|
const insert = handle.prepare(`INSERT INTO leaderboard_matches
|
|
(server, game, map, map_key, map_name, started_at, last_received_at, status,
|
|
ended_at, server_deleted_at, highest_round, last_event, last_fingerprint)
|
|
VALUES (?, 't5', 'zombie_theater', 't5:zombie_theater', 'Kino Der Toten',
|
|
'2026-10-01T00:00:00Z', '2026-10-01T01:00:00Z', ?, ?, ?, 20, 'heartbeat', 'test')`);
|
|
insert.run(job.name, 'active', null, null);
|
|
insert.run(job.name, 'completed', '2026-10-01T01:00:00Z', null);
|
|
insert.run(job.name, 'interrupted', '2026-10-01T01:00:00Z', null);
|
|
insert.run(job.name, 'active', null, '2026-10-01T01:00:00Z');
|
|
insert.run('another-server', 'active', null, null);
|
|
const matches = () => handle.prepare('SELECT * FROM leaderboard_matches ORDER BY id').all();
|
|
const before = matches();
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const stopped = requestLifecycle(job.name, 'stop');
|
|
if (!stopped.ok) throw new Error(stopped.error);
|
|
await advance();
|
|
expect(matches()).toEqual(before);
|
|
return { stopped, matches, before };
|
|
}
|
|
|
|
it.each(['stopped', 'exited', 'created', 'dead'])(
|
|
'interrupts only the current open match after verifying the container is %s',
|
|
async (state) => {
|
|
const { stopped, matches, before } = await setupStop();
|
|
// A successful command receipt alone does not prove the container stopped.
|
|
await acknowledge(stopped.deploymentId);
|
|
expect(matches()).toEqual(before);
|
|
remote.state = state;
|
|
const verifiedAfter = Date.now();
|
|
await advance();
|
|
expect((await detail(stopped.deploymentId)).state).toBe('done');
|
|
const after = matches();
|
|
expect(after[0]).toEqual({
|
|
...(before[0] as object),
|
|
status: 'interrupted',
|
|
ended_at: expect.any(String)
|
|
});
|
|
const endedAt = Date.parse((after[0] as { ended_at: string }).ended_at);
|
|
expect(endedAt).toBeGreaterThanOrEqual(verifiedAfter);
|
|
expect(endedAt).toBeLessThanOrEqual(Date.now());
|
|
expect(after.slice(1)).toEqual(before.slice(1));
|
|
await advance();
|
|
expect(matches()).toEqual(after);
|
|
}
|
|
);
|
|
|
|
it.each(['failed', 'unknown'] as const)(
|
|
'keeps the match open when the stop command is %s',
|
|
async (state) => {
|
|
const { stopped, matches, before } = await setupStop();
|
|
const operation = (await operationFor(stopped.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, { state, error: 'Stop not confirmed' });
|
|
await advance();
|
|
expect(matches()).toEqual(before);
|
|
}
|
|
);
|
|
});
|
|
|
|
describe('game key pool launch lifecycle', () => {
|
|
it('reserves across maps atomically, rejects duplicate credentials, and reuses a cancelled launch key', async () => {
|
|
await prepareLibrary();
|
|
const { saveServerKey, serverKeyPool, getServerKey } =
|
|
await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { queueCreateServer, destroyServer, previewCreation } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
saveServerKey('t5', 'shared-t5-key', 'Any T5 map');
|
|
saveServerKey('t6', 'shared-t6-key');
|
|
const input = {
|
|
game: 't5' as const,
|
|
gameMode: 'zombie' as const,
|
|
mapSlug: 'moon',
|
|
hostId,
|
|
libraryId,
|
|
image
|
|
};
|
|
expect(previewCreation(input).hasSavedKey).toBe(true);
|
|
const first = queueCreateServer(input);
|
|
if (!first.ok) throw new Error(first.message);
|
|
expect(first.name).toBe('t5_zombie_moon_1');
|
|
expect(getServerKey(first.name)).toBe('shared-t5-key');
|
|
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 0 });
|
|
expect(serverKeyPool('t6')).toEqual({ total: 1, available: 1 });
|
|
expect(previewCreation({ ...input, mapSlug: 'theater' }).hasSavedKey).toBe(false);
|
|
expect(queueCreateServer({ ...input, mapSlug: 'theater' }).ok).toBe(false);
|
|
expect(
|
|
queueCreateServer({ ...input, mapSlug: 'theater', plutoKey: 'shared-t5-key' }).ok
|
|
).toBe(false);
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
expect(serverKeyPool('t5').available).toBe(0);
|
|
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1);
|
|
expect(destroyServer(first.name).ok).toBe(true);
|
|
expect(serverKeyPool('t5').available).toBe(1);
|
|
const second = queueCreateServer({ ...input, mapSlug: 'theater' });
|
|
if (!second.ok) throw new Error(second.message);
|
|
expect(getServerKey(second.name)).toBe('shared-t5-key');
|
|
expect(getServerKey(first.name)).toBeUndefined();
|
|
});
|
|
|
|
it('keeps a key through stop and rebuild, then releases it only after confirmed removal', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { serverKeyPool, getServerKey } =
|
|
await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const stopped = requestLifecycle(job.name, 'stop');
|
|
if (!stopped.ok) throw new Error(stopped.error);
|
|
await advance();
|
|
remote.state = 'stopped';
|
|
await acknowledge(stopped.deploymentId);
|
|
expect(serverKeyPool('t5').available).toBe(0);
|
|
const rebuilt = requestLifecycle(job.name, 'rebuild');
|
|
if (!rebuilt.ok) throw new Error(rebuilt.error);
|
|
await advance();
|
|
remote.state = 'running';
|
|
await acknowledge(rebuilt.deploymentId);
|
|
expect(getServerKey(job.name)).toBe('saved-key');
|
|
expect(serverKeyPool('t5').available).toBe(0);
|
|
const removal = destroyServer(job.name);
|
|
if (!removal.ok || !removal.deploymentId) throw new Error('Expected removal');
|
|
await advance();
|
|
expect(serverKeyPool('t5').available).toBe(0);
|
|
const operation = (await operationFor(removal.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, { state: 'unknown', error: 'Receipt lost' });
|
|
await advance();
|
|
expect(serverKeyPool('t5').available).toBe(0);
|
|
await acknowledge(removal.deploymentId);
|
|
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
|
|
expect(getServerKey(job.name)).toBeUndefined();
|
|
});
|
|
|
|
it('keeps the reservation for a submitted cancellation until the host confirms cleanup', async () => {
|
|
const job = await queue();
|
|
await advance();
|
|
const { serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
expect(cleanup).toEqual(expect.any(String));
|
|
expect(serverKeyPool('t5').available).toBe(0);
|
|
await acknowledge(cleanup);
|
|
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
|
|
});
|
|
});
|
|
|
|
describe('completed rebuild generation receipts (#155)', () => {
|
|
async function rebuild() {
|
|
const job = await queue();
|
|
await advance();
|
|
await acknowledge(job.deploymentId);
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const requested = requestLifecycle(job.name, 'rebuild');
|
|
if (!requested.ok) throw new Error(requested.error);
|
|
await advance();
|
|
return {
|
|
job,
|
|
id: requested.deploymentId,
|
|
operation: (await operationFor(requested.deploymentId))!
|
|
};
|
|
}
|
|
|
|
async function inventoryGeneration(
|
|
instanceId: string,
|
|
generation: number,
|
|
observedAt = new Date().toISOString(),
|
|
targetHost = hostId
|
|
) {
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(targetHost, {
|
|
...inventory(),
|
|
instanceStates: [
|
|
{ instanceId, generation, state: 'running', healthy: true, observedAt }
|
|
]
|
|
});
|
|
}
|
|
|
|
it('repairs an older job binding from the actual command before verification, including duplicate receipts and restart', async () => {
|
|
const { job, id } = await rebuild();
|
|
await acknowledge(id);
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const next = requestLifecycle(job.name, 'rebuild');
|
|
if (!next.ok) throw new Error(next.error);
|
|
await advance();
|
|
const operation = (await operationFor(next.deploymentId))!;
|
|
expect(operation.generation).toBe(2);
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
// Historical job snapshots could stay on generation 1 while the submitted
|
|
// command correctly targeted 2 -> 3. The old completion path kept 2.
|
|
db()
|
|
.prepare('UPDATE native_provisioning SET generation=1 WHERE deployment_id=?')
|
|
.run(next.deploymentId);
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { instanceId: operation.instanceId, generation: 3, healthy: true }
|
|
});
|
|
await inventoryGeneration(operation.instanceId!, 3);
|
|
closeDb();
|
|
const { nativeRequest, nativeInstance } =
|
|
await import('../../src/lib/server/backends/native.js');
|
|
vi.mocked(nativeRequest).mockClear();
|
|
remote.state = 'starting';
|
|
await advance();
|
|
expect(nativeInstance(job.name)?.generation).toBe(3);
|
|
expect(nativeRequest).toHaveBeenCalledWith(
|
|
expect.objectContaining({ id: operation.instanceId, generation: 3 }),
|
|
'instance.inspect'
|
|
);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT generation,applied_at FROM native_provisioning WHERE deployment_id=?'
|
|
)
|
|
.get(next.deploymentId)
|
|
).toEqual({ generation: 2, applied_at: expect.any(String) });
|
|
const { reconcileNativeProvisioningReceipt } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
const before = db()
|
|
.prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
|
|
.get(operation.instanceId);
|
|
reconcileNativeProvisioningReceipt(next.deploymentId);
|
|
closeDb();
|
|
reconcileNativeProvisioningReceipt(next.deploymentId);
|
|
expect(nativeInstance(job.name)?.generation).toBe(3);
|
|
expect(
|
|
db()
|
|
.prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
|
|
.get(operation.instanceId)
|
|
).toEqual(before);
|
|
expect((await detail(next.deploymentId)).state).toBe('awaiting-health');
|
|
for (const observation of await observations(job.name))
|
|
expect(observation.health).not.toBe('online');
|
|
const { latestGameSubmission, latestGameSubmissions } =
|
|
await import('../../src/lib/server/fleet/lifecycle-evidence.js');
|
|
expect(latestGameSubmission(job.name)).toBe(
|
|
(before as { generation_reconciled_at: string }).generation_reconciled_at
|
|
);
|
|
expect(latestGameSubmissions().get(job.name)).toBe(latestGameSubmission(job.name));
|
|
remote.state = 'running';
|
|
await advance();
|
|
expect((await detail(next.deploymentId)).state).toBe('done');
|
|
const restart = requestLifecycle(job.name, 'restart');
|
|
if (!restart.ok) throw new Error(restart.error);
|
|
await advance();
|
|
expect(await operationFor(restart.deploymentId)).toMatchObject({ generation: 3 });
|
|
});
|
|
|
|
it.each(['before', 'after'] as const)(
|
|
'reconciles delayed completion reported %s cancellation across restart',
|
|
async (when) => {
|
|
const { job, id, operation } = await rebuild();
|
|
const { reportHostOperation, pendingHostOperations } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
pendingHostOperations(hostId);
|
|
if (when === 'before')
|
|
reportHostOperation(hostId, operation.id, { state: 'completed' });
|
|
expect(cancelProvisioningJob(id)).toBe(true);
|
|
closeDb();
|
|
if (when === 'after') reportHostOperation(hostId, operation.id, { state: 'completed' });
|
|
await advance();
|
|
await advance();
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.generation).toBe(2);
|
|
expect((await detail(id)).state).toBe('cancelled');
|
|
}
|
|
);
|
|
|
|
it('recovers an already-applied cancelled mismatch through the API and targets replacement logs, health and removal', async () => {
|
|
const { job, id, operation } = await rebuild();
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { instanceId: operation.instanceId, generation: 2 }
|
|
});
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
expect(cancelProvisioningJob(id)).toBe(true);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?')
|
|
.run(operation.updatedAt, id);
|
|
await inventoryGeneration(operation.instanceId!, 2);
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
expect(getDeployment(id)?.generationRecovery).toMatchObject({
|
|
expectedGeneration: 1,
|
|
observedGeneration: 2,
|
|
available: true
|
|
});
|
|
const { POST } =
|
|
await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
|
|
expect(await POST({ params: { id } }).json()).toEqual({ ok: true, generation: 2 });
|
|
expect(POST({ params: { id } }).status).toBe(200);
|
|
expect((await detail(id)).state).toBe('cancelled');
|
|
expect(getDeployment(id)?.generationRecovery).toBeNull();
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
vi.mocked(backend.nativeRequest).mockRestore();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
vi.spyOn(hosts, 'waitHostOperationStatus').mockImplementation(async (id) =>
|
|
hosts.reportHostOperationStatus(hostId, id, {
|
|
state: 'completed',
|
|
result: { healthy: true, state: 'running' }
|
|
})
|
|
);
|
|
await backend.nativeRequest(backend.nativeInstance(job.name)!, 'logs.read', {
|
|
file: 'console.log'
|
|
});
|
|
expect(await backend.nativeHealth(job.name)).toMatchObject({ ok: true });
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT generation FROM ec_host_operations WHERE kind IN ('logs.read','instance.inspect')"
|
|
)
|
|
.all()
|
|
.every((row) => (row as { generation: number }).generation === 2)
|
|
).toBe(true);
|
|
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const removal = destroyServer(job.name);
|
|
if (!removal.ok || !removal.deploymentId) throw new Error('Removal not queued');
|
|
await advance();
|
|
expect(await operationFor(removal.deploymentId)).toMatchObject({
|
|
generation: 2,
|
|
kind: 'instance.remove'
|
|
});
|
|
});
|
|
|
|
it.each([
|
|
'host',
|
|
'instance',
|
|
'result-instance',
|
|
'result-generation',
|
|
'payload',
|
|
'stale-inventory',
|
|
'missing-inventory',
|
|
'inventory-instance'
|
|
] as const)('rejects recovery evidence with %s mismatch', async (mismatch) => {
|
|
const { job, id, operation } = await rebuild();
|
|
const { reportHostOperation, createHost } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: {
|
|
instanceId:
|
|
mismatch === 'result-instance' ? 'another-instance' : operation.instanceId,
|
|
generation: mismatch === 'result-generation' ? 3 : 2
|
|
}
|
|
});
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
if (mismatch === 'host')
|
|
db()
|
|
.prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?')
|
|
.run(createHost({ name: 'Other host' }).id, operation.id);
|
|
if (mismatch === 'instance')
|
|
db()
|
|
.prepare('UPDATE ec_host_operations SET instance_id=? WHERE id=?')
|
|
.run('another-instance', operation.id);
|
|
if (mismatch === 'payload')
|
|
db()
|
|
.prepare(
|
|
"UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?"
|
|
)
|
|
.run(operation.id);
|
|
if (mismatch !== 'missing-inventory')
|
|
await inventoryGeneration(
|
|
mismatch === 'inventory-instance' ? 'another-instance' : operation.instanceId!,
|
|
2,
|
|
mismatch === 'stale-inventory' ? '2000-01-01T00:00:00Z' : new Date().toISOString()
|
|
);
|
|
const { POST } =
|
|
await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
|
|
expect(POST({ params: { id } }).status).toBe(409);
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.generation).toBe(1);
|
|
});
|
|
|
|
it('pauses after an agent stale-generation rejection even when inventory is unavailable', async () => {
|
|
const { id, operation } = await rebuild();
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, { state: 'completed' });
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
vi.mocked(nativeRequest).mockClear();
|
|
vi.mocked(nativeRequest).mockRejectedValueOnce(
|
|
new Error('Instance generation has been superseded or removed')
|
|
);
|
|
await advance();
|
|
expect((await detail(id)).state).toBe('interrupted');
|
|
expect((await detail(id)).error).toContain('controller expects 2, host reports unknown');
|
|
await advance();
|
|
expect(nativeRequest).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('stops verification retries on a confirmed generation mismatch and keeps expected/observed evidence', async () => {
|
|
const { id, operation } = await rebuild();
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, { state: 'completed' });
|
|
await inventoryGeneration(operation.instanceId!, 3);
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
vi.mocked(nativeRequest).mockClear();
|
|
await advance();
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const before = getDeployment(id)!;
|
|
expect(before.deployment.state).toBe('interrupted');
|
|
expect(before.progress?.blocker).toMatchObject({
|
|
code: 'generation-recovery',
|
|
automatic: false,
|
|
message: expect.stringContaining(
|
|
'controller expects 1, completed rebuild requests 2, host reports 3'
|
|
)
|
|
});
|
|
await advance();
|
|
await advance();
|
|
expect(nativeRequest).not.toHaveBeenCalled();
|
|
expect(getDeployment(id)?.events).toEqual(before.events);
|
|
});
|
|
});
|