Reconcile recovered campaign targets (Refs #166)

2026-10-19 20:22:58 -04:00
parent 0f05be3a59
commit b5ed0831ae
13 changed files with 802 additions and 21 deletions
@@ -1948,6 +1948,145 @@ describe('native instance provisioning', () => {
        expect(nativeInstance(job.name)?.generation).toBe(3);
    });

+    it.each(['pending', 'applied'])(
+        'retries %s files from a failed campaign on the current generation through occupancy and startup admission',
+        async (fileState) => {
+            const job = await queue();
+            await advance();
+            await acknowledge(job.deploymentId);
+            const { db } = await import('../../src/lib/server/db/index.js');
+            const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
+            const { requestRestart } =
+                await import('../../src/lib/server/provisioning/requests.js');
+            const { finishDeployment } =
+                await import('../../src/lib/server/provisioning/history.js');
+            const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+            const { processProvisioning } =
+                await import('../../src/lib/server/provisioning/worker.js');
+            db()
+                .prepare(
+                    "INSERT INTO deployments(id,action,started_at) VALUES('retry','promote',?)"
+                )
+                .run(new Date().toISOString());
+            db()
+                .prepare(
+                    "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('retry',?,1,'failed')"
+                )
+                .run(job.name);
+            queueFileChange({
+                sourceKind: 'campaign',
+                sourceId: 'retry',
+                server: job.name,
+                kind: 'script',
+                relPath: 'retry.gsc',
+                content: Buffer.from('desired campaign bytes'),
+                inventorySource: 'test'
+            });
+            const failed = requestRestart(job.name, { kind: 'campaign', id: 'retry' });
+            finishDeployment(failed.id, { ok: false, message: 'Download timeout' });
+            db()
+                .prepare(
+                    "UPDATE staged_file_changes SET state=? WHERE source_kind='campaign' AND source_id='retry'"
+                )
+                .run(fileState);
+            const { queueNativeRebuild } =
+                await import('../../src/lib/server/provisioning/native.js');
+            const unrelated = queueNativeRebuild(job.name, 'ui');
+            if (!unrelated.ok) throw new Error(unrelated.message);
+            await advance();
+            await acknowledge(unrelated.deploymentId);
+            await sweep();
+            expect(
+                db()
+                    .prepare(
+                        "SELECT restart_state FROM deployment_servers WHERE deployment_id='retry'"
+                    )
+                    .get()
+            ).toEqual({ restart_state: 'failed' });
+
+            const { fleetSettings, saveFleetSettings } =
+                await import('../../src/lib/server/settings.js');
+            saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
+            const { queueCreateServer } =
+                await import('../../src/lib/server/provisioning/instances.js');
+            const other = queueCreateServer({
+                game: 't5',
+                gameMode: 'zombie',
+                mapSlug: 'theater',
+                hostId,
+                libraryId,
+                image,
+                plutoKey: 'second-key'
+            });
+            if (!other.ok) throw new Error(other.message);
+            await advance();
+            expect(await operationFor(other.deploymentId)).not.toBeNull();
+
+            db()
+                .prepare("UPDATE deployment_servers SET restarted_at=? WHERE deployment_id='retry'")
+                .run(new Date().toISOString());
+            const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js');
+            expect(POST({ params: { id: 'retry' } }).status).toBe(202);
+            await sweep();
+            const next = requestRestart(job.name, { kind: 'campaign', id: 'retry' });
+            expect(next).toMatchObject({ action: 'rebuild', state: 'waiting', submitted_at: null });
+            expect(next.id).not.toBe(failed.id);
+            expect(
+                db()
+                    .prepare('SELECT generation FROM native_provisioning WHERE deployment_id=?')
+                    .get(next.id)
+            ).toEqual({ generation: 2 });
+            await processProvisioning();
+            const staged = db()
+                .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
+                .get(next.id) as { stage_operation_id: string };
+            const { reportHostOperation, getHostOperation } =
+                await import('../../src/lib/server/hosts/index.js');
+            reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
+            await processProvisioning();
+            expect(await operationFor(next.id)).toBeNull();
+            expect((await detail(next.id)).stage).toContain('startup capacity');
+            remote.players = 2;
+            await acknowledge(other.deploymentId);
+            await processProvisioning();
+            expect(await operationFor(next.id)).toBeNull();
+            expect((await detail(next.id)).stage).toContain('player');
+            remote.players = 0;
+            await processProvisioning();
+            const activation = (await operationFor(next.id))!;
+            expect(activation, (await detail(next.id)).stage).toMatchObject({
+                kind: 'instance.rebuild',
+                generation: 2,
+                payload: { nextGeneration: 3, stageId: next.id }
+            });
+            const files = getHostOperation(staged.stage_operation_id)!.payload.files as {
+                id: number;
+                path: string;
+            }[];
+            reportHostOperation(hostId, activation.id, {
+                state: 'completed',
+                result: { before: files.map((file) => ({ ...file, contentBase64: null })) }
+            });
+            await processProvisioning();
+            await sweep();
+            expect(
+                db()
+                    .prepare(
+                        "SELECT restart_state,provisioning_id FROM deployment_servers WHERE deployment_id='retry'"
+                    )
+                    .get()
+            ).toEqual({ restart_state: 'done', provisioning_id: next.id });
+            expect((await detail(failed.id)).state).toBe('failed');
+            expect(
+                db()
+                    .prepare(
+                        "SELECT operation_id FROM provisioning_sources WHERE kind='campaign' AND source_id='retry' ORDER BY rowid"
+                    )
+                    .all()
+            ).toEqual([{ operation_id: failed.id }, { operation_id: next.id }]);
+        }
+    );
+
    it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
        const job = await queue();
        await advance();