Fix native provisioning scheduling and progress reporting

2026-10-04 14:50:59 -04:00
parent 1bbee98b62
commit 2adadb0d05
18 changed files with 569 additions and 84 deletions
@@ -44,6 +44,37 @@ afterEach(async () => {
 });

 describe('reviewed drift plans', () => {
+    it('reads files in bounded batches and preserves deterministic scan order', async () => {
+        const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
+        await seedServerRegistry();
+        const { destOfKind } = await import('../../src/lib/server/config.js');
+        const destination = destOfKind('t5_zombie_theater_1', 'plugin');
+        const names = Array.from({ length: 12 }, (_, i) => `batch-${String(i).padStart(2, '0')}.dll`);
+        await Promise.all(names.map(name => writeFile(join(destination, name), name)));
+        const files = await import('../../src/lib/server/backends/files.js');
+        const original = files.readFile;
+        let release!: () => void;
+        const gate = new Promise<void>(resolve => { release = resolve; });
+        let active = 0; let peak = 0;
+        const read = vi.spyOn(files, 'readFile').mockImplementation(async path => {
+            if (!path.includes('batch-')) return original(path);
+            peak = Math.max(peak, ++active);
+            try { await gate; return await original(path); }
+            finally { active--; }
+        });
+        const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
+        const scan = scanServerDrift('t5_zombie_theater_1');
+        try {
+            await vi.waitFor(() => expect(active).toBe(4));
+            expect(read.mock.calls.filter(([path]) => path.includes('batch-'))).toHaveLength(4);
+            release();
+            const result = await scan;
+            expect(result.filter(entry => entry.relPath.startsWith('batch-')).map(entry => entry.relPath)).toEqual(names);
+            expect(peak).toBe(4);
+            expect(active).toBe(0);
+        } finally { release(); await scan; read.mockRestore(); }
+    });
+
    it('keeps live deletions, replacements, and inventory unchanged until activation', async () => {
        const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
@@ -73,6 +73,129 @@ async function detail(id: string) {
 }

 describe('native instance provisioning', () => {
+    it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => {
+        const job = await queue();
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const { isProvisioningDetail, provisioningGroups } = await import('../../src/lib/ui/provisioning-presentation.js');
+        const queued = getDeployment(job.deploymentId)!;
+        expect(queued.progress?.phase).toBe('eligibility');
+        expect(queued.progress?.blocker).toBeNull();
+        expect(queued.backend).toBe('native');
+        await advance();
+        expect(getDeployment(job.deploymentId)!.progress?.recipeStatus).toBe('attempted');
+        await acknowledge(job.deploymentId);
+        const result = getDeployment(job.deploymentId)!;
+        expect(result.deployment.state).toBe('done');
+        expect(isProvisioningDetail(result, job.deploymentId)).toBe(true);
+        expect(result.progress?.checks.map(check => check.key)).toEqual(['container', 'game', 'map', 'plutonium-release', 'plutonium-version']);
+        expect(result.progress?.checks.every(check => check.result === 'pass')).toBe(true);
+        expect(result.progress?.checks[0].source).toBe('agent');
+        const steps = provisioningGroups(result).flatMap(group => group.steps);
+        expect(steps.every(step => step.state === 'done')).toBe(true);
+        expect(steps.some(step => step.label.includes('Komodo'))).toBe(false);
+        expect(result.agentOperation?.state).toBe('completed');
+    });
+
+    it('records agent receipts immediately and only once, even before the provisioning worker runs', async () => {
+        const job = await queue(); await advance();
+        const operation = (await operationFor(job.deploymentId))!;
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        for (const state of ['accepted', 'accepted', 'running', 'completed', 'completed']) {
+            reportHostOperation(hostId, operation.id, { state, result: { state: 'running', healthy: true } });
+        }
+        const result = getDeployment(job.deploymentId)!;
+        expect(result.deployment.state).toBe('running');
+        expect(result.agentOperation?.state).toBe('completed');
+        expect(result.events.filter(event => event.kind === 'agent')).toHaveLength(3);
+        expect(result.progress?.checkpoints.map(point => point.key)).toEqual(expect.arrayContaining(['agent-accepted', 'agent-running', 'agent-completed']));
+        expect(result.progress?.command?.state).toBe('accepted');
+    });
+
+    it('records an outstanding container check before awaiting the host and admits other jobs meanwhile', async () => {
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
+        const first = await queue(); await advance();
+        const operation = (await operationFor(first.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
+        const second = await queue();
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        let release!: (value: never) => void;
+        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
+        const pending = advance();
+        try {
+            const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+            expect(getDeployment(first.deploymentId)!.progress?.phase).toBe('verify-container');
+            expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
+        } finally {
+            release({ healthy: true, state: 'running' } as never);
+            await pending;
+        }
+    });
+
+    it('admits a later request on the next scheduler pass while an earlier native probe is pending', async () => {
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
+        const first = await queue(); await advance();
+        const operation = (await operationFor(first.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        let release!: (value: never) => void;
+        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning({ wait: false });
+        try {
+            const second = await queue();
+            await processProvisioning({ wait: false });
+            expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
+            expect(vi.mocked(nativeRequest)).toHaveBeenCalledTimes(1);
+        } finally {
+            release({ healthy: true, state: 'running' } as never);
+            await processProvisioning();
+        }
+    });
+
+    it('ignores a late health result after removal retires the operation', async () => {
+        const job = await queue(); await advance();
+        const operation = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        let release!: (value: never) => void;
+        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
+        const pending = advance();
+        try {
+            await vi.waitFor(() => expect(release).toBeTypeOf('function'));
+            const { db } = await import('../../src/lib/server/db/index.js');
+            db().prepare("UPDATE server_deployments SET state='cancelled',server_deleted_at=? WHERE id=?")
+                .run(new Date().toISOString(), job.deploymentId);
+            const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+            const before = getDeployment(job.deploymentId)!;
+            release({ healthy: true, state: 'running' } as never);
+            await pending;
+            const after = getDeployment(job.deploymentId)!;
+            expect(after.deployment.state).toBe('cancelled');
+            expect(after.progress).toEqual(before.progress);
+            expect(after.activity).toEqual(before.activity);
+        } finally { release({ healthy: true, state: 'running' } as never); await pending; }
+    });
+
+    it('records map mismatch evidence while retaining passed container and game checks', async () => {
+        const job = await queue(); await advance(); remote.map = 'zombie_moon';
+        await acknowledge(job.deploymentId);
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const result = getDeployment(job.deploymentId)!;
+        expect(result.deployment.state).toBe('awaiting-health');
+        expect(result.progress?.phase).toBe('verify-map');
+        expect(result.progress?.checks).toEqual(expect.arrayContaining([
+            expect.objectContaining({ key: 'container', result: 'pass' }),
+            expect.objectContaining({ key: 'game', result: 'pass' }),
+            expect.objectContaining({ key: 'map', result: 'waiting', actual: 'zombie_moon', expected: 'zombie_theater' })
+        ]));
+    });
+
    it('selects the latest game files automatically and rejects stale explicit selections', async () => {
        const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
        expect(() => nativeSelection({ hostId, image }, 't5')).toThrow('Upload T5 game files');
@@ -0,0 +1,76 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/pipeline/sweep.js', () => ({ sweep: vi.fn() }));
+vi.mock('../../src/lib/server/pipeline/beta-runner.js', () => ({ betaTick: vi.fn() }));
+vi.mock('../../src/lib/server/maintenance/index.js', () => ({ runMaintenance: vi.fn() }));
+vi.mock('../../src/lib/server/provisioning/worker.js', () => ({ processProvisioning: vi.fn(async () => {}) }));
+
+function deferred() {
+    let resolve!: () => void;
+    const promise = new Promise<void>(done => { resolve = done; });
+    return { promise, resolve };
+}
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); });
+afterEach(async () => { await fixture.cleanup(); });
+
+async function setup() {
+    const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+    const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
+    const { runMaintenance } = await import('../../src/lib/server/maintenance/index.js');
+    const background = deferred();
+    vi.mocked(sweep).mockImplementation(() => background.promise as never);
+    vi.mocked(betaTick).mockImplementation(() => background.promise as never);
+    vi.mocked(runMaintenance).mockImplementation(() => background.promise);
+    const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+    const worker = await import('../../src/lib/server/pipeline/worker.js');
+    const { workerActivity } = await import('../../src/lib/server/pipeline/activity.js');
+    return { ...worker, workerActivity, background, process: vi.mocked(processProvisioning), sweep, betaTick, runMaintenance };
+}
+
+describe('independent provisioning scheduling', () => {
+    it('admits and checks new work while campaign, beta and maintenance observers are still waiting', async () => {
+        const s = await setup();
+        try {
+            await s.kick();
+            expect(s.process).toHaveBeenCalledTimes(1);
+            expect(s.workerActivity().map(work => work.key)).toEqual(expect.arrayContaining(['campaign', 'beta', 'maintenance']));
+            await s.kick();
+            expect(s.process).toHaveBeenCalledTimes(2);
+            expect(s.sweep).toHaveBeenCalledTimes(1);
+            expect(s.betaTick).toHaveBeenCalledTimes(1);
+            expect(s.runMaintenance).toHaveBeenCalledTimes(1);
+        } finally {
+            s.background.resolve();
+            await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
+        }
+    });
+
+    it('coalesces overlapping kicks but makes another pass for requests arriving during execution', async () => {
+        const s = await setup();
+        const busy = deferred();
+        s.process.mockImplementationOnce(() => busy.promise);
+        const first = s.kick();
+        const second = s.kick();
+        const receipt = s.kickProvisioning();
+        try {
+            expect(s.process).toHaveBeenCalledTimes(1);
+            busy.resolve();
+            await Promise.all([first, second, receipt]);
+            expect(s.process).toHaveBeenCalledTimes(2);
+        } finally {
+            busy.resolve(); s.background.resolve();
+            await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
+        }
+    });
+
+    it('wakes provisioning when a background observer finishes submitting its requests', async () => {
+        const s = await setup();
+        await s.kick();
+        s.process.mockClear();
+        s.background.resolve();
+        await vi.waitFor(() => expect(s.process).toHaveBeenCalled());
+        await vi.waitFor(() => expect(s.workerActivity()).toEqual([]));
+    });
+});