Add Prettier and ESLint tooling (Refs #77)
This commit is contained in:
+350
-104
@@ -3,32 +3,64 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } })),
|
||||
rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
|
||||
rconStatusResult: vi.fn(async () => ({
|
||||
ok: true,
|
||||
endpoint: { host: 'agent', port: 28960 },
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
||||
})),
|
||||
rconGetPlayerCount: vi.fn(async () => 0),
|
||||
rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({
|
||||
kick: vi.fn(async () => {}),
|
||||
kickProvisioning: vi.fn(async () => {})
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture; let hostId: string; let instanceId: string;
|
||||
let fixture: Fixture;
|
||||
let hostId: string;
|
||||
let instanceId: string;
|
||||
const server = 't5_zombie_theater_1';
|
||||
const healthy = { exists: true, healthy: true, state: 'running' };
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]);
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
|
||||
const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { createHost, createEnrollment, enrollHost } =
|
||||
await import('../../src/lib/server/hosts/index.js');
|
||||
hostId = createHost({ name: 'Safety test host' }).id;
|
||||
enrollHost(createEnrollment(hostId).token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000, runtimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }], capabilities: ['instance.safety-override'] });
|
||||
enrollHost(createEnrollment(hostId).token, {
|
||||
protocolVersion: 1,
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'published-game-version', state: 'ready' }],
|
||||
capabilities: ['instance.safety-override']
|
||||
});
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
instanceId = randomUUID();
|
||||
db().prepare("INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'t5','zombie_theater','zombie','theater',28960,'running')").run(server);
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'published-game-version','plutainer:test',28960,?)`).run(instanceId, server, hostId, new Date().toISOString());
|
||||
db()
|
||||
.prepare(
|
||||
"INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'t5','zombie_theater','zombie','theater',28960,'running')"
|
||||
)
|
||||
.run(server);
|
||||
db()
|
||||
.prepare(
|
||||
`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'published-game-version','plutainer:test',28960,?)`
|
||||
)
|
||||
.run(instanceId, server, hostId, new Date().toISOString());
|
||||
const backend = await import('../../src/lib/server/backends/native.js');
|
||||
vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => {
|
||||
if (kind !== 'instance.inspect') throw new Error(`Unexpected remote mutation: ${kind}`);
|
||||
return healthy as never;
|
||||
});
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); });
|
||||
afterEach(async () => {
|
||||
await fixture.cleanup();
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
async function advance() {
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
@@ -36,34 +68,65 @@ async function advance() {
|
||||
}
|
||||
async function operationId(id: string) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
return (db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(id) as { operation_id: string | null }).operation_id;
|
||||
return (
|
||||
db()
|
||||
.prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?')
|
||||
.get(id) as { operation_id: string | null }
|
||||
).operation_id;
|
||||
}
|
||||
async function waitingRestart() {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const result = requestLifecycle(server, 'restart'); if (!result.ok) throw new Error(result.error);
|
||||
const result = requestLifecycle(server, 'restart');
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
return result.deploymentId;
|
||||
}
|
||||
async function suspendInspection() {
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
let release!: () => void;
|
||||
vi.mocked(nativeRequest).mockImplementationOnce(async () => { await new Promise<void>(resolve => { release = resolve; }); return healthy as never; });
|
||||
vi.mocked(nativeRequest).mockImplementationOnce(async () => {
|
||||
await new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
return healthy as never;
|
||||
});
|
||||
const running = advance();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
return { running, release };
|
||||
}
|
||||
async function stageCampaign() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign-race','promote',?)").run(new Date().toISOString());
|
||||
db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign-race',?,1,'pending')").run(server);
|
||||
db()
|
||||
.prepare(
|
||||
"INSERT INTO deployments(id,action,started_at) VALUES('campaign-race','promote',?)"
|
||||
)
|
||||
.run(new Date().toISOString());
|
||||
db()
|
||||
.prepare(
|
||||
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign-race',?,1,'pending')"
|
||||
)
|
||||
.run(server);
|
||||
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign-race', server, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('reviewed campaign bytes'), inventorySource: 'campaign:test' });
|
||||
queueFileChange({
|
||||
sourceKind: 'campaign',
|
||||
sourceId: 'campaign-race',
|
||||
server,
|
||||
kind: 'script',
|
||||
relPath: 'hud.gsc',
|
||||
content: Buffer.from('reviewed campaign bytes'),
|
||||
inventorySource: 'campaign:test'
|
||||
});
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const restart = requestRestart(server, { kind: 'campaign', id: 'campaign-race' });
|
||||
await advance();
|
||||
const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string };
|
||||
const staged = db()
|
||||
.prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
|
||||
.get(restart.id) as { stage_operation_id: string };
|
||||
expect(staged.stage_operation_id).toBeTruthy();
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed', result: { state: 'staged' } });
|
||||
reportHostOperation(hostId, staged.stage_operation_id, {
|
||||
state: 'completed',
|
||||
result: { state: 'staged' }
|
||||
});
|
||||
return restart.id;
|
||||
}
|
||||
|
||||
@@ -71,92 +134,192 @@ describe('native disruption safety at dispatch', () => {
|
||||
it('lets fresh occupied telemetry veto an empty RCON result and resumes after that evidence expires', async () => {
|
||||
const id = await waitingRestart();
|
||||
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
||||
saveTelemetry({ event: 'game_start', server, map: 'zombie_theater', players: [{ name: 'Still playing', guid: 'player-1' }], receivedAt: new Date().toISOString() });
|
||||
saveTelemetry({
|
||||
event: 'game_start',
|
||||
server,
|
||||
map: 'zombie_theater',
|
||||
players: [{ name: 'Still playing', guid: 'player-1' }],
|
||||
receivedAt: new Date().toISOString()
|
||||
});
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 });
|
||||
db().prepare('UPDATE server_telemetry SET received_at=? WHERE server=?').run('2000-01-01T00:00:00Z', server);
|
||||
expect(
|
||||
db()
|
||||
.prepare(
|
||||
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'"
|
||||
)
|
||||
.get()
|
||||
).toEqual({ count: 0 });
|
||||
db()
|
||||
.prepare('UPDATE server_telemetry SET received_at=? WHERE server=?')
|
||||
.run('2000-01-01T00:00:00Z', server);
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeTypeOf('string');
|
||||
});
|
||||
|
||||
it('holds destruction if a beta becomes active while remote inspection is pending', async () => {
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const removal = destroyServer(server); if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued.');
|
||||
const removal = destroyServer(server);
|
||||
if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued.');
|
||||
const pending = await suspendInspection();
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { script, version } = createScript({ name: 'Race beta', kind: 'script', targetRelPath: 'beta.gsc', content: Buffer.from('main() {}'), scopeType: 'game', scopeValue: 't5' });
|
||||
const { script, version } = createScript({
|
||||
name: 'Race beta',
|
||||
kind: 'script',
|
||||
targetRelPath: 'beta.gsc',
|
||||
content: Buffer.from('main() {}'),
|
||||
scopeType: 'game',
|
||||
scopeValue: 't5'
|
||||
});
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const betaId = db().prepare("INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')").run(script.id, version.id).lastInsertRowid;
|
||||
db().prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')").run(betaId, server);
|
||||
pending.release(); await pending.running;
|
||||
const betaId = db()
|
||||
.prepare(
|
||||
"INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')"
|
||||
)
|
||||
.run(script.id, version.id).lastInsertRowid;
|
||||
db()
|
||||
.prepare(
|
||||
"INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')"
|
||||
)
|
||||
.run(betaId, server);
|
||||
pending.release();
|
||||
await pending.running;
|
||||
expect(await operationId(removal.deploymentId)).toBeNull();
|
||||
expect(db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toEqual({ name: server });
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.remove'").get()).toEqual({ count: 0 });
|
||||
expect(db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toEqual({
|
||||
name: server
|
||||
});
|
||||
expect(
|
||||
db()
|
||||
.prepare(
|
||||
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.remove'"
|
||||
)
|
||||
.get()
|
||||
).toEqual({ count: 0 });
|
||||
});
|
||||
|
||||
it.each(['cancel', 'change bytes'] as const)('does not activate staged files after a %s during remote inspection', async change => {
|
||||
const id = await stageCampaign();
|
||||
const pending = await suspendInspection();
|
||||
const { cancelStagedFiles, queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
if (change === 'cancel') cancelStagedFiles('campaign', 'campaign-race', server);
|
||||
else queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign-race', server, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('different unreviewed bytes'), inventorySource: 'campaign:test' });
|
||||
pending.release(); await pending.running;
|
||||
expect(await operationId(id)).toBeNull();
|
||||
expect(stagedFileChanges('campaign', 'campaign-race')[0].state).toBe(change === 'cancel' ? 'cancelled' : 'pending');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 });
|
||||
});
|
||||
it.each(['cancel', 'change bytes'] as const)(
|
||||
'does not activate staged files after a %s during remote inspection',
|
||||
async (change) => {
|
||||
const id = await stageCampaign();
|
||||
const pending = await suspendInspection();
|
||||
const { cancelStagedFiles, queueFileChange, stagedFileChanges } =
|
||||
await import('../../src/lib/server/fleet/staged-files.js');
|
||||
if (change === 'cancel') cancelStagedFiles('campaign', 'campaign-race', server);
|
||||
else
|
||||
queueFileChange({
|
||||
sourceKind: 'campaign',
|
||||
sourceId: 'campaign-race',
|
||||
server,
|
||||
kind: 'script',
|
||||
relPath: 'hud.gsc',
|
||||
content: Buffer.from('different unreviewed bytes'),
|
||||
inventorySource: 'campaign:test'
|
||||
});
|
||||
pending.release();
|
||||
await pending.running;
|
||||
expect(await operationId(id)).toBeNull();
|
||||
expect(stagedFileChanges('campaign', 'campaign-race')[0].state).toBe(
|
||||
change === 'cancel' ? 'cancelled' : 'pending'
|
||||
);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(
|
||||
db()
|
||||
.prepare(
|
||||
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'"
|
||||
)
|
||||
.get()
|
||||
).toEqual({ count: 0 });
|
||||
}
|
||||
);
|
||||
|
||||
it('does not dispatch an old generation if the instance changes during inspection', async () => {
|
||||
const id = await waitingRestart();
|
||||
const pending = await suspendInspection();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId);
|
||||
pending.release(); await pending.running;
|
||||
pending.release();
|
||||
await pending.running;
|
||||
expect(await operationId(id)).toBeNull();
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 });
|
||||
expect(
|
||||
db()
|
||||
.prepare(
|
||||
"SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'"
|
||||
)
|
||||
.get()
|
||||
).toEqual({ count: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('provisioning admin override', () => {
|
||||
const actor = { id: 'admin-id', username: 'recovery-admin' };
|
||||
async function confirm(id: string) {
|
||||
const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
|
||||
const { enableSafetyOverride } =
|
||||
await import('../../src/lib/server/provisioning/safety-override.js');
|
||||
expect(enableSafetyOverride(id, actor)).toEqual({ ok: true });
|
||||
}
|
||||
|
||||
it.each(['restart', 'stop', 'destroy'] as const)('dispatches %s with unavailable player evidence only after confirmation', async action => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const result = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, action);
|
||||
if (!result.ok || !result.deploymentId) throw new Error('Request failed');
|
||||
const id = result.deploymentId;
|
||||
const status = await import('../../src/lib/server/fleet/status-evidence.js');
|
||||
vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ players: null, source: null, observedAt: new Date().toISOString() });
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
expect(getDeployment(id)).toMatchObject({ canOverrideSafety: true, safetyOverride: null });
|
||||
await confirm(id);
|
||||
await confirm(id);
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb(); // Approval and attribution survive controller restart.
|
||||
await advance();
|
||||
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const commandId = (await operationId(id))!;
|
||||
expect(getHostOperation(commandId)).toMatchObject({ kind: `instance.${action === 'destroy' ? 'remove' : action}`, payload: { safetyOverride: true } });
|
||||
expect(getDeployment(id)).toMatchObject({ canOverrideSafety: false, safetyOverride: { actorId: actor.id, actorName: actor.username } });
|
||||
expect(db().prepare("SELECT COUNT(*) n FROM server_deployment_events WHERE deployment_id=? AND kind='safety-override'").get(id)).toEqual({ n: 1 });
|
||||
await advance();
|
||||
expect(await operationId(id)).toBe(commandId);
|
||||
});
|
||||
it.each(['restart', 'stop', 'destroy'] as const)(
|
||||
'dispatches %s with unavailable player evidence only after confirmation',
|
||||
async (action) => {
|
||||
const { requestLifecycle } =
|
||||
await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { destroyServer } =
|
||||
await import('../../src/lib/server/provisioning/instances.js');
|
||||
const result =
|
||||
action === 'destroy' ? destroyServer(server) : requestLifecycle(server, action);
|
||||
if (!result.ok || !result.deploymentId) throw new Error('Request failed');
|
||||
const id = result.deploymentId;
|
||||
const status = await import('../../src/lib/server/fleet/status-evidence.js');
|
||||
vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({
|
||||
players: null,
|
||||
source: null,
|
||||
observedAt: new Date().toISOString()
|
||||
});
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
expect(getDeployment(id)).toMatchObject({
|
||||
canOverrideSafety: true,
|
||||
safetyOverride: null
|
||||
});
|
||||
await confirm(id);
|
||||
await confirm(id);
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb(); // Approval and attribution survive controller restart.
|
||||
await advance();
|
||||
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const commandId = (await operationId(id))!;
|
||||
expect(getHostOperation(commandId)).toMatchObject({
|
||||
kind: `instance.${action === 'destroy' ? 'remove' : action}`,
|
||||
payload: { safetyOverride: true }
|
||||
});
|
||||
expect(getDeployment(id)).toMatchObject({
|
||||
canOverrideSafety: false,
|
||||
safetyOverride: { actorId: actor.id, actorName: actor.username }
|
||||
});
|
||||
expect(
|
||||
db()
|
||||
.prepare(
|
||||
"SELECT COUNT(*) n FROM server_deployment_events WHERE deployment_id=? AND kind='safety-override'"
|
||||
)
|
||||
.get(id)
|
||||
).toEqual({ n: 1 });
|
||||
await advance();
|
||||
expect(await operationId(id)).toBe(commandId);
|
||||
}
|
||||
);
|
||||
|
||||
it('bypasses occupied telemetry while retaining health verification after the restart', async () => {
|
||||
const id = await waitingRestart();
|
||||
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
||||
saveTelemetry({ event: 'game_start', server, map: 'zombie_theater', players: [{ name: 'Playing', guid: '1' }], receivedAt: new Date().toISOString() });
|
||||
saveTelemetry({
|
||||
event: 'game_start',
|
||||
server,
|
||||
map: 'zombie_theater',
|
||||
players: [{ name: 'Playing', guid: '1' }],
|
||||
receivedAt: new Date().toISOString()
|
||||
});
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
await confirm(id);
|
||||
@@ -168,20 +331,26 @@ describe('provisioning admin override', () => {
|
||||
await advance();
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
expect(getDeployment(id)?.deployment.state).toBe('awaiting-health');
|
||||
expect(getDeployment(id)?.progress?.checks).toContainEqual(expect.objectContaining({ key: 'container', result: 'waiting' }));
|
||||
expect(getDeployment(id)?.progress?.checks).toContainEqual(
|
||||
expect.objectContaining({ key: 'container', result: 'waiting' })
|
||||
);
|
||||
});
|
||||
|
||||
it('retains recovery capacity limits and cancellation after approval', async () => {
|
||||
const id = await waitingRestart();
|
||||
await confirm(id);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
const { fleetSettings, saveFleetSettings } =
|
||||
await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 });
|
||||
db().prepare("INSERT INTO servers(name) VALUES('other')").run();
|
||||
db().prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('other',?)").run(new Date().toISOString());
|
||||
db()
|
||||
.prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('other',?)")
|
||||
.run(new Date().toISOString());
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { cancelProvisioningJob } =
|
||||
await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
db().prepare("DELETE FROM restart_rollout WHERE server='other'").run();
|
||||
await advance();
|
||||
@@ -191,16 +360,37 @@ describe('provisioning admin override', () => {
|
||||
it('rejects old agents, stale instances, unsupported actions and submitted or terminal operations', async () => {
|
||||
const id = await waitingRestart();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { enableSafetyOverride, safetyOverrideAvailability } = await import('../../src/lib/server/provisioning/safety-override.js');
|
||||
db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json('[]')) WHERE id=?").run(hostId);
|
||||
expect(enableSafetyOverride(id, actor)).toMatchObject({ ok: false, error: expect.stringContaining('Update') });
|
||||
for (const state of ['running', 'awaiting-health', 'failed', 'interrupted', 'cancelled', 'done']) {
|
||||
const { enableSafetyOverride, safetyOverrideAvailability } =
|
||||
await import('../../src/lib/server/provisioning/safety-override.js');
|
||||
db()
|
||||
.prepare(
|
||||
"UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json('[]')) WHERE id=?"
|
||||
)
|
||||
.run(hostId);
|
||||
expect(enableSafetyOverride(id, actor)).toMatchObject({
|
||||
ok: false,
|
||||
error: expect.stringContaining('Update')
|
||||
});
|
||||
for (const state of [
|
||||
'running',
|
||||
'awaiting-health',
|
||||
'failed',
|
||||
'interrupted',
|
||||
'cancelled',
|
||||
'done'
|
||||
]) {
|
||||
db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, id);
|
||||
expect(safetyOverrideAvailability(id).applicable).toBe(false);
|
||||
}
|
||||
db().prepare("UPDATE server_deployments SET state='waiting',action='start' WHERE id=?").run(id);
|
||||
db()
|
||||
.prepare("UPDATE server_deployments SET state='waiting',action='start' WHERE id=?")
|
||||
.run(id);
|
||||
expect(safetyOverrideAvailability(id).applicable).toBe(false);
|
||||
db().prepare("UPDATE server_deployments SET action='restart',submitted_at='2026-10-05' WHERE id=?").run(id);
|
||||
db()
|
||||
.prepare(
|
||||
"UPDATE server_deployments SET action='restart',submitted_at='2026-10-05' WHERE id=?"
|
||||
)
|
||||
.run(id);
|
||||
expect(safetyOverrideAvailability(id).applicable).toBe(false);
|
||||
db().prepare('UPDATE server_deployments SET submitted_at=NULL WHERE id=?').run(id);
|
||||
db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId);
|
||||
@@ -212,37 +402,61 @@ describe('provisioning admin override', () => {
|
||||
const id = await waitingRestart();
|
||||
const pending = await suspendInspection();
|
||||
await confirm(id);
|
||||
pending.release(); await pending.running;
|
||||
pending.release();
|
||||
await pending.running;
|
||||
expect(await operationId(id)).toBeNull();
|
||||
await advance();
|
||||
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ safetyOverride: true });
|
||||
expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({
|
||||
safetyOverride: true
|
||||
});
|
||||
});
|
||||
|
||||
it('overrides a waiting campaign restart while keeping its staged changes attached', async () => {
|
||||
const id = await stageCampaign();
|
||||
const status = await import('../../src/lib/server/fleet/status-evidence.js');
|
||||
vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ players: null, source: null, observedAt: new Date().toISOString() });
|
||||
vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({
|
||||
players: null,
|
||||
source: null,
|
||||
observedAt: new Date().toISOString()
|
||||
});
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
await confirm(id);
|
||||
await advance();
|
||||
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ stageId: id, safetyOverride: true });
|
||||
expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({
|
||||
stageId: id,
|
||||
safetyOverride: true
|
||||
});
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_id='campaign-race'").get()).toEqual({ state: 'applying' });
|
||||
expect(
|
||||
db()
|
||||
.prepare("SELECT state FROM staged_file_changes WHERE source_id='campaign-race'")
|
||||
.get()
|
||||
).toEqual({ state: 'applying' });
|
||||
});
|
||||
|
||||
it('requires an administrator and explicit confirmation at the API boundary', async () => {
|
||||
const id = await waitingRestart();
|
||||
const { POST } = await import('../../src/routes/api/provisioning/[id]/override/+server.js');
|
||||
const request = (role: string | null, confirmed: unknown) => POST({ params: { id }, locals: { user: role ? { ...actor, role } : null },
|
||||
request: new Request(`http://localhost/api/provisioning/${id}/override`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ confirmed }) })
|
||||
} as Parameters<typeof POST>[0]);
|
||||
for (const role of [null, 'viewer', 'operator']) expect((await request(role, true)).status).toBe(403);
|
||||
for (const value of [undefined, false, 'true', 1]) expect((await request('admin', value)).status).toBe(400);
|
||||
const request = (role: string | null, confirmed: unknown) =>
|
||||
POST({
|
||||
params: { id },
|
||||
locals: { user: role ? { ...actor, role } : null },
|
||||
request: new Request(`http://localhost/api/provisioning/${id}/override`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ confirmed })
|
||||
})
|
||||
} as Parameters<typeof POST>[0]);
|
||||
for (const role of [null, 'viewer', 'operator'])
|
||||
expect((await request(role, true)).status).toBe(403);
|
||||
for (const value of [undefined, false, 'true', 1])
|
||||
expect((await request('admin', value)).status).toBe(400);
|
||||
expect((await request('admin', true)).status).toBe(202);
|
||||
const { safetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
|
||||
const { safetyOverride } =
|
||||
await import('../../src/lib/server/provisioning/safety-override.js');
|
||||
expect(safetyOverride(id)?.actorId).toBe(actor.id);
|
||||
});
|
||||
});
|
||||
@@ -253,20 +467,34 @@ describe('native scheduler ownership', () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
db().prepare("INSERT INTO servers(name) VALUES('unplaced-registry')").run();
|
||||
expect(requestLifecycle('unplaced-registry', 'restart')).toMatchObject({ ok: false, status: 503 });
|
||||
expect(requestLifecycle('unplaced-registry', 'restart')).toMatchObject({
|
||||
ok: false,
|
||||
status: 503
|
||||
});
|
||||
expect(destroyServer('unplaced-registry')).toMatchObject({ ok: false, kind: 'conflict' });
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE server='unplaced-registry'").all()).toEqual([]);
|
||||
expect(
|
||||
db().prepare("SELECT id FROM server_deployments WHERE server='unplaced-registry'").all()
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('shares restart capacity until an agent receipt and fresh health release the first slot', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
const { fleetSettings, saveFleetSettings } =
|
||||
await import('../../src/lib/server/settings.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 });
|
||||
db().prepare("INSERT INTO servers(name,game,map,game_port) VALUES('second','t5','zombie_theater',28961)").run();
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
SELECT 'second-instance','second',host_id,library_id,image,28961,created_at FROM native_instances WHERE id=?`).run(instanceId);
|
||||
db()
|
||||
.prepare(
|
||||
"INSERT INTO servers(name,game,map,game_port) VALUES('second','t5','zombie_theater',28961)"
|
||||
)
|
||||
.run();
|
||||
db()
|
||||
.prepare(
|
||||
`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
SELECT 'second-instance','second',host_id,library_id,image,28961,created_at FROM native_instances WHERE id=?`
|
||||
)
|
||||
.run(instanceId);
|
||||
const first = await waitingRestart();
|
||||
const second = requestLifecycle('second', 'restart');
|
||||
if (!second.ok) throw new Error(second.error);
|
||||
@@ -280,22 +508,40 @@ describe('native scheduler ownership', () => {
|
||||
reportHostOperation(hostId, command!, { state: 'completed', result: healthy });
|
||||
await advance();
|
||||
await advance();
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(first)).toEqual({ state: 'done' });
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(first)).toEqual({
|
||||
state: 'done'
|
||||
});
|
||||
expect(await operationId(second.deploymentId)).toEqual(expect.any(String));
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toHaveLength(2);
|
||||
expect(
|
||||
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()
|
||||
).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('holds a paused campaign and cancels withdrawn work before delivering a restart', async () => {
|
||||
const id = await stageCampaign();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE deployment_servers SET restart_state='held' WHERE deployment_id='campaign-race'").run();
|
||||
db()
|
||||
.prepare(
|
||||
"UPDATE deployment_servers SET restart_state='held' WHERE deployment_id='campaign-race'"
|
||||
)
|
||||
.run();
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'waiting' });
|
||||
db().prepare("UPDATE deployment_servers SET restart_state='done' WHERE deployment_id='campaign-race'").run();
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({
|
||||
state: 'waiting'
|
||||
});
|
||||
db()
|
||||
.prepare(
|
||||
"UPDATE deployment_servers SET restart_state='done' WHERE deployment_id='campaign-race'"
|
||||
)
|
||||
.run();
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' });
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]);
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({
|
||||
state: 'cancelled'
|
||||
});
|
||||
expect(
|
||||
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user