246 lines
10 KiB
TypeScript
246 lines
10 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { STATUS_REFRESH_MS } from '../../src/lib/server-freshness.js';
|
|
|
|
vi.mock('../../src/lib/server/pipeline/sweep.js', () => ({ sweep: vi.fn() }));
|
|
vi.mock('../../src/lib/server/pipeline/beta-runner.js', () => ({ betaTick: vi.fn() }));
|
|
vi.mock('../../src/lib/server/maintenance/index.js', () => ({ runMaintenance: vi.fn() }));
|
|
vi.mock('../../src/lib/server/game-files/index.js', () => ({
|
|
cleanupControllerLibraries: vi.fn(async () => {})
|
|
}));
|
|
vi.mock('../../src/lib/server/provisioning/worker.js', () => ({
|
|
processProvisioning: vi.fn(async () => {})
|
|
}));
|
|
vi.mock('../../src/lib/server/retention.js', () => ({ drainRetention: vi.fn(async () => 0) }));
|
|
vi.mock('../../src/lib/server/fleet/chat.js', () => ({ collectFleetChat: vi.fn(async () => {}) }));
|
|
|
|
function deferred() {
|
|
let resolve!: () => void;
|
|
const promise = new Promise<void>((done) => {
|
|
resolve = done;
|
|
});
|
|
return { promise, resolve };
|
|
}
|
|
let fixture: Fixture;
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.clearAllMocks();
|
|
fixture = await newFixture([]);
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
});
|
|
|
|
async function setup() {
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
const { runMaintenance } = await import('../../src/lib/server/maintenance/index.js');
|
|
const background = deferred();
|
|
vi.mocked(sweep).mockImplementation(() => background.promise as never);
|
|
vi.mocked(betaTick).mockImplementation(() => background.promise as never);
|
|
vi.mocked(runMaintenance).mockImplementation(() => background.promise);
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const worker = await import('../../src/lib/server/pipeline/worker.js');
|
|
const { workerActivity } = await import('../../src/lib/server/pipeline/activity.js');
|
|
return {
|
|
...worker,
|
|
workerActivity,
|
|
background,
|
|
process: vi.mocked(processProvisioning),
|
|
sweep,
|
|
betaTick,
|
|
runMaintenance
|
|
};
|
|
}
|
|
|
|
describe('independent provisioning scheduling', () => {
|
|
it.each(['startup', 'scheduled'] as const)(
|
|
'logs a failed %s status snapshot and retries on the next interval',
|
|
async (phase) => {
|
|
const s = await setup();
|
|
const { MonitoringSnapshot } =
|
|
await import('../../src/lib/server/fleet/monitoring-snapshot.js');
|
|
const refresh = vi.spyOn(MonitoringSnapshot.prototype, 'refresh');
|
|
const log = vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
const failure = new Error('database or disk is full');
|
|
vi.useFakeTimers();
|
|
try {
|
|
if (phase === 'scheduled') {
|
|
s.startWorker();
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
refresh.mockClear();
|
|
}
|
|
refresh.mockImplementationOnce(() => {
|
|
throw failure;
|
|
});
|
|
if (phase === 'startup') s.startWorker();
|
|
await vi.advanceTimersByTimeAsync(phase === 'startup' ? 0 : STATUS_REFRESH_MS);
|
|
expect(refresh).toHaveBeenCalledTimes(1);
|
|
expect(log).toHaveBeenCalledExactlyOnceWith(
|
|
'[worker] status fanout error:',
|
|
failure
|
|
);
|
|
|
|
await vi.advanceTimersByTimeAsync(STATUS_REFRESH_MS);
|
|
expect(refresh).toHaveBeenCalledTimes(2);
|
|
expect(log).toHaveBeenCalledTimes(1);
|
|
} finally {
|
|
s.stopWorker();
|
|
s.background.resolve();
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
vi.useRealTimers();
|
|
refresh.mockRestore();
|
|
log.mockRestore();
|
|
await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
|
|
}
|
|
}
|
|
);
|
|
|
|
it('collects match chat on worker ticks without overlapping a slow collection', async () => {
|
|
const s = await setup();
|
|
const { collectFleetChat } = await import('../../src/lib/server/fleet/chat.js');
|
|
const collection = deferred();
|
|
vi.mocked(collectFleetChat).mockImplementationOnce(() => collection.promise);
|
|
vi.useFakeTimers();
|
|
try {
|
|
s.startWorker();
|
|
expect(collectFleetChat).toHaveBeenCalledTimes(1);
|
|
await vi.advanceTimersByTimeAsync(30_000);
|
|
expect(collectFleetChat).toHaveBeenCalledTimes(1);
|
|
collection.resolve();
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
await vi.advanceTimersByTimeAsync(15_000);
|
|
expect(collectFleetChat).toHaveBeenCalledTimes(2);
|
|
} finally {
|
|
s.stopWorker();
|
|
collection.resolve();
|
|
s.background.resolve();
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
vi.useRealTimers();
|
|
await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
|
|
}
|
|
});
|
|
|
|
it('runs retention at most every five minutes across ticks and explicit kicks', async () => {
|
|
const s = await setup();
|
|
const { drainRetention } = await import('../../src/lib/server/retention.js');
|
|
vi.useFakeTimers();
|
|
try {
|
|
s.startWorker();
|
|
expect(drainRetention).toHaveBeenCalledTimes(1);
|
|
await vi.advanceTimersByTimeAsync(285_000);
|
|
await s.kick();
|
|
expect(drainRetention).toHaveBeenCalledTimes(1);
|
|
await vi.advanceTimersByTimeAsync(15_000);
|
|
expect(drainRetention).toHaveBeenCalledTimes(2);
|
|
expect(s.process.mock.calls.length).toBeGreaterThan(2);
|
|
} finally {
|
|
s.stopWorker();
|
|
s.background.resolve();
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
vi.useRealTimers();
|
|
await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
|
|
}
|
|
});
|
|
|
|
it('keeps a retention backlog running without overlapping it or blocking provisioning', async () => {
|
|
const s = await setup();
|
|
const { drainRetention } = await import('../../src/lib/server/retention.js');
|
|
const cleanup = deferred();
|
|
vi.mocked(drainRetention).mockImplementationOnce(async () => {
|
|
await cleanup.promise;
|
|
return 1501;
|
|
});
|
|
vi.useFakeTimers();
|
|
try {
|
|
s.startWorker();
|
|
await vi.advanceTimersByTimeAsync(600_000);
|
|
expect(drainRetention).toHaveBeenCalledTimes(1);
|
|
expect(s.process.mock.calls.length).toBeGreaterThan(2);
|
|
cleanup.resolve();
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
} finally {
|
|
s.stopWorker();
|
|
cleanup.resolve();
|
|
s.background.resolve();
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
vi.useRealTimers();
|
|
await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
|
|
}
|
|
});
|
|
|
|
it('runs controller library cleanup on worker ticks without overlapping a slow cleanup', async () => {
|
|
const s = await setup();
|
|
const { cleanupControllerLibraries } =
|
|
await import('../../src/lib/server/game-files/index.js');
|
|
const cleanup = deferred();
|
|
vi.mocked(cleanupControllerLibraries).mockImplementationOnce(() => cleanup.promise);
|
|
vi.useFakeTimers();
|
|
try {
|
|
s.startWorker();
|
|
expect(cleanupControllerLibraries).toHaveBeenCalledTimes(1);
|
|
await vi.advanceTimersByTimeAsync(30_000);
|
|
expect(cleanupControllerLibraries).toHaveBeenCalledTimes(1);
|
|
expect(s.process.mock.calls.length).toBeGreaterThan(1);
|
|
cleanup.resolve();
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
await vi.advanceTimersByTimeAsync(15_000);
|
|
expect(cleanupControllerLibraries).toHaveBeenCalledTimes(2);
|
|
} finally {
|
|
s.stopWorker();
|
|
cleanup.resolve();
|
|
s.background.resolve();
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
vi.useRealTimers();
|
|
await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
|
|
}
|
|
});
|
|
|
|
it('admits and checks new work while campaign, beta and maintenance observers are still waiting', async () => {
|
|
const s = await setup();
|
|
try {
|
|
await s.kick();
|
|
expect(s.process).toHaveBeenCalledTimes(1);
|
|
expect(s.workerActivity().map((work) => work.key)).toEqual(
|
|
expect.arrayContaining(['campaign', 'beta', 'maintenance'])
|
|
);
|
|
await s.kick();
|
|
expect(s.process).toHaveBeenCalledTimes(2);
|
|
expect(s.sweep).toHaveBeenCalledTimes(1);
|
|
expect(s.betaTick).toHaveBeenCalledTimes(1);
|
|
expect(s.runMaintenance).toHaveBeenCalledTimes(1);
|
|
} finally {
|
|
s.background.resolve();
|
|
await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
|
|
}
|
|
});
|
|
|
|
it('coalesces overlapping kicks but makes another pass for requests arriving during execution', async () => {
|
|
const s = await setup();
|
|
const busy = deferred();
|
|
s.process.mockImplementationOnce(() => busy.promise);
|
|
const first = s.kick();
|
|
const second = s.kick();
|
|
const receipt = s.kickProvisioning();
|
|
try {
|
|
expect(s.process).toHaveBeenCalledTimes(1);
|
|
busy.resolve();
|
|
await Promise.all([first, second, receipt]);
|
|
expect(s.process).toHaveBeenCalledTimes(2);
|
|
} finally {
|
|
busy.resolve();
|
|
s.background.resolve();
|
|
await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
|
|
}
|
|
});
|
|
|
|
it('wakes provisioning when a background observer finishes submitting its requests', async () => {
|
|
const s = await setup();
|
|
await s.kick();
|
|
s.process.mockClear();
|
|
s.background.resolve();
|
|
await vi.waitFor(() => expect(s.process).toHaveBeenCalled());
|
|
await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
|
|
});
|
|
});
|