josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild beta and campaign file activations

Browse Source
This commit is contained in:
josh
2026-10-12 22:52:58 -04:00
parent fa0baa8db5
commit 537b9841a7
22 changed files with 533 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/beta-plans.test.ts
+3 -3
View File
@@ -22,7 +22,7 @@ vi.mock('../../src/lib/server/provisioning/worker.js', () => ({
await import('../../src/lib/server/fleet/status-evidence.js');
const jobs = db()
.prepare(
"SELECT id,server,state FROM server_deployments WHERE action='restart' AND state IN ('waiting','awaiting-health')"
"SELECT id,server,state FROM server_deployments WHERE action IN ('restart','rebuild') AND state IN ('waiting','awaiting-health')"
)
.all() as { id: string; server: string; state: string }[];
for (const job of jobs) {
@@ -243,7 +243,7 @@ describe('durable beta test plans', () => {
expect(getScript(file.script_id)?.current_version_id).toBe(replacement);
});
it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => {
it('waits for occupied telemetry to clear before rebuilding a beta despite an empty RCON response', async () => {
const id = await openPlan(defaultPlan());
const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
await postTelemetry(gameEvent('heartbeat', 4, 2));
@@ -437,7 +437,7 @@ describe('durable beta test plans', () => {
await expect(promoteBeta(id)).rejects.toThrow();
});
it('automatically rolls back only the deployed wave and queues safe rollback restarts', async () => {
it('automatically rolls back only the deployed wave and queues safe rollback rebuilds', async () => {
await writeFile(scriptPath(names[0]), 'original');
const id = await openPlan(
{
tests/unit/beta-staging.test.ts
+7 -1
View File
@@ -290,7 +290,13 @@ describe('staged beta files', () => {
if (blocked)
await expect(request).rejects.toMatchObject({
kind: 'conflict',
details: [expect.objectContaining({ deploymentId: promotion.deploymentId })]
...(operationState === 'waiting'
? { message: expect.stringContaining('rebuild') }
: {
details: [
expect.objectContaining({ deploymentId: promotion.deploymentId })
]
})
});
else await expect(request).resolves.toMatchObject({ servers: [servers[1]] });
}
tests/unit/campaign-resume.test.ts
+23
View File
@@ -128,6 +128,29 @@ describe('campaign resume API', () => {
}
);
it('upgrades queued file restarts without replaying submitted operations', async () => {
await heldRollback('pending');
const { db } = await import('../../src/lib/server/db/index.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const job = requestRestart(server, { kind: 'campaign', id: campaign });
expect(job.action).toBe('rebuild');
db().prepare("UPDATE server_deployments SET action='restart' WHERE id=?").run(job.id);
expect(requestRestart(server, { kind: 'campaign', id: campaign })).toMatchObject({
id: job.id,
action: 'rebuild'
});
db()
.prepare(
"UPDATE server_deployments SET action='restart',state='running',submitted_at=? WHERE id=?"
)
.run(new Date().toISOString(), job.id);
expect(requestRestart(server, { kind: 'campaign', id: campaign })).toMatchObject({
id: job.id,
action: 'restart',
state: 'running'
});
});
it('retains an existing waiting operation when its held campaign resumes', async () => {
await heldRollback();
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
tests/unit/native-provisioning.test.ts
+121 -3
View File
@@ -39,7 +39,7 @@ const inventory = () => ({
memoryTotal: 64 * 1024 ** 3,
memoryAvailable: 60 * 1024 ** 3,
protocolVersion: 1,
capabilities: ['instance.unhealthy-recovery'],
capabilities: ['instance.unhealthy-recovery', 'instance.rebuild-files'],
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100_000_000_000,
@@ -1725,6 +1725,96 @@ describe('native instance provisioning', () => {
const { getHost } = await import('../../src/lib/server/hosts/index.js');
expect(getHost(hostId)?.mode).toBe('active');
});
it('builds campaign replacements with requested plugins, configuration, additions and removals', async () => {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { db } = await import('../../src/lib/server/db/index.js');
for (const path of ['replace.dll', 'remove.dll']) {
const file = createScript({
name: path,
kind: 'plugin',
targetRelPath: path,
scopeType: 'game',
scopeValue: 't5',
content: Buffer.from('promoted plugin')
});
db()
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
.run(file.version.id, file.script.id);
}
const job = await queue();
await advance();
await acknowledge(job.deploymentId);
db()
.prepare(
"INSERT INTO deployments(id,action,started_at) VALUES('files-rebuild','promote',?)"
)
.run(new Date().toISOString());
db()
.prepare(
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('files-rebuild',?,4,'pending')"
)
.run(job.name);
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
const changes = [
{ kind: 'plugin', relPath: 'replace.dll', content: Buffer.from('requested plugin') },
{ kind: 'plugin', relPath: 'remove.dll', content: null },
{ kind: 'script', relPath: 'new.gsc', content: Buffer.from('new script') },
{ kind: 'config', relPath: 'custom.cfg', content: Buffer.from('set custom 1') }
] as const;
for (const change of changes)
queueFileChange({
sourceKind: 'campaign',
sourceId: 'files-rebuild',
server: job.name,
inventorySource: 'campaign',
...change
});
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const rebuild = requestRestart(job.name, { kind: 'campaign', id: 'files-rebuild' });
await advance();
const { getHostOperation, reportHostOperation } =
await import('../../src/lib/server/hosts/index.js');
const staged = db()
.prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
.get(rebuild.id) as { stage_operation_id: string };
reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
await advance();
const operation = (await operationFor(rebuild.id))!;
expect(operation.kind).toBe('instance.rebuild');
expect(operation.payload).not.toHaveProperty('unhealthyRecovery');
const { recipeForDeployment } =
await import('../../src/lib/server/fleet/launch-recipes.js');
const recipe = recipeForDeployment(rebuild.id)!.recipe;
for (const change of changes) {
const actual = recipe.files.find(
(file) => file.kind === change.kind && file.path === change.relPath
);
if (change.content === null) expect(actual).toBeUndefined();
else expect(Buffer.from(actual!.content!, 'base64')).toEqual(change.content);
}
const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{
id: number;
path: string;
}>;
reportHostOperation(hostId, operation.id, {
state: 'completed',
result: { before: files.map((file) => ({ ...file, contentBase64: null })) }
});
remote.state = 'unhealthy';
await advance();
expect((await operationFor(rebuild.id))!.kind).toBe('instance.restart');
await acknowledge(rebuild.id);
const recovery = (await operationFor(rebuild.id))!;
expect(recovery.kind).toBe('instance.rebuild');
expect(recovery.payload).not.toHaveProperty('stageId');
expect(recovery.payload.files).toEqual(operation.payload.files);
remote.state = 'running';
await acknowledge(rebuild.id);
expect((await detail(rebuild.id)).state).toBe('done');
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
expect(nativeInstance(job.name)?.generation).toBe(3);
});
it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
const job = await queue();
await advance();
@@ -1761,6 +1851,19 @@ describe('native instance provisioning', () => {
});
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const restart = requestRestart(job.name, { kind: 'campaign', id: 'campaign' });
expect(restart.action).toBe('rebuild');
// Upgrade queued work from older controllers, but wait for a capable host.
db().prepare("UPDATE server_deployments SET action='restart' WHERE id=?").run(restart.id);
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), capabilities: [] });
await advance();
expect(await detail(restart.id)).toMatchObject({
action: 'rebuild',
state: 'waiting',
stage: expect.stringContaining('Update the host agent')
});
expect(await operationFor(restart.id)).toBeNull();
heartbeat(hostId, inventory());
remote.players = 2;
await advance();
const staged = db()
@@ -1811,9 +1914,18 @@ describe('native instance provisioning', () => {
await advance();
const activation = (await operationFor(restart.id))!;
expect(activation).toMatchObject({
kind: 'instance.restart',
kind: 'instance.rebuild',
generation: 1,
payload: { stageId: restart.id }
payload: {
stageId: restart.id,
nextGeneration: 2,
files: expect.arrayContaining([
expect.objectContaining({
path: 'runtime/plutonium/storage/t5/raw/scripts/sp/hud.gsc',
sha256: createHash('sha256').update(after).digest('hex')
})
])
}
});
expect(
db()
@@ -3819,6 +3931,12 @@ describe('cancelled native mutation bookkeeping', () => {
])
).rejects.toThrow();
const activation = (await operationFor(restart.id))!;
expect(activation.kind).toBe('instance.rebuild');
expect(activation.payload).toMatchObject({
stageId: restart.id,
nextGeneration: activation.generation! + 1
});
expect(activation.payload).not.toHaveProperty('unhealthyRecovery');
reportHostOperation(hostId, activation.id, {
state: 'completed',
result: {
tests/unit/native-safety.test.ts
+24 -6
View File
@@ -36,7 +36,7 @@ beforeEach(async () => {
storageAvailable: 100_000_000_000,
preparedRuntimeImages: ['plutainer:test'],
libraries: [{ id: 'published-game-version', state: 'ready' }],
capabilities: ['instance.safety-override']
capabilities: ['instance.safety-override', 'instance.rebuild-files']
});
const { db } = await import('../../src/lib/server/db/index.js');
instanceId = randomUUID();
@@ -105,6 +105,22 @@ async function stageCampaign() {
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign-race',?,1,'pending')"
)
.run(server);
const { createLibraryImport, updateHostLibraryCaches } =
await import('../../src/lib/server/game-files/index.js');
const library = createLibraryImport({ hostId, game: 't5' });
db()
.prepare(
"UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?"
)
.run('a'.repeat(64), new Date().toISOString(), library.id);
updateHostLibraryCaches(hostId, [
{ libraryVersionId: library.id, status: 'ready', manifestSha256: 'a'.repeat(64) }
]);
db().prepare('UPDATE native_instances SET library_id=? WHERE server=?').run(library.id, server);
const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
reserveServerKey(server, 't5', 'saved-key');
const backend = await import('../../src/lib/server/backends/native.js');
vi.spyOn(backend, 'runtimePasswordForServer').mockReturnValue('saved-password');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
queueFileChange({
sourceKind: 'campaign',
@@ -413,7 +429,7 @@ describe('provisioning admin override', () => {
});
});
it('overrides a waiting campaign restart while keeping its staged changes attached', async () => {
it('overrides a waiting campaign rebuild while keeping its staged changes attached', async () => {
const id = await stageCampaign();
const status = await import('../../src/lib/server/fleet/status-evidence.js');
vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({
@@ -426,9 +442,11 @@ describe('provisioning admin override', () => {
await confirm(id);
await advance();
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({
stageId: id,
safetyOverride: true
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'running', error: null });
expect(getHostOperation((await operationId(id))!)).toMatchObject({
kind: 'instance.rebuild',
payload: { stageId: id, safetyOverride: true }
});
const { db } = await import('../../src/lib/server/db/index.js');
expect(
@@ -518,7 +536,7 @@ describe('native scheduler ownership', () => {
).toHaveLength(2);
});
it('holds a paused campaign and cancels withdrawn work before delivering a restart', async () => {
it('holds a paused campaign and cancels withdrawn work before delivering a rebuild', async () => {
const id = await stageCampaign();
const { db } = await import('../../src/lib/server/db/index.js');
db()