josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Offer rebuild retry for failed creation commands

Browse Source
This commit is contained in:
josh
2026-10-08 14:05:38 -04:00
parent 511f090552
commit e01b143a09
12 changed files with 298 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/native-provisioning.test.ts
+163
View File
@@ -1503,6 +1503,169 @@ describe('automatic capacity admission and placement', () => {
});
});
describe('native verification resume and explicit failed-creation retry', () => {
async function failedCreation(state: 'failed' | 'interrupted' = 'failed') {
const job = await queue(); await advance();
const original = (await operationFor(job.deploymentId))!;
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
reportHostOperation(hostId, original.id, { state: 'failed', error: 'Host runtime preflight failed' });
await advance();
if (state === 'interrupted') {
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId);
}
return { job, original };
}
it.each(['failed', 'interrupted'] as const)('retries a %s creation through a new rebuild while retaining the original failure', async state => {
const { job, original } = await failedCreation(state);
const history = await import('../../src/lib/server/provisioning/history.js');
expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: true });
const { POST: resume } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
expect(resume({ params: { id: job.deploymentId } }).status).toBe(409);
const { POST: retry } = await import('../../src/routes/api/provisioning/[id]/retry-rebuild/+server.js');
const response = await retry({ params: { id: job.deploymentId } });
expect(response.status).toBe(202);
const next = await response.json() as { ok: boolean; deploymentId: string };
expect(next.ok).toBe(true);
expect(next.deploymentId).not.toBe(job.deploymentId);
expect(await detail(next.deploymentId)).toMatchObject({ action: 'rebuild', state: 'waiting', server: job.name });
expect(history.getDeployment(job.deploymentId)).toMatchObject({
canRetryWithRebuild: false,
deployment: { state, error: 'Host runtime preflight failed' },
progress: { recovery: { state: 'transferred', operationId: next.deploymentId } }
});
expect((await retry({ params: { id: job.deploymentId } })).status).toBe(409);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server=?').get(job.name)).toEqual({ count: 2 });
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
expect(nativeInstance(job.name)).toMatchObject({ id: original.instanceId, generation: 1, port: original.payload.port });
remote.state = 'missing'; await advance();
const rebuilding = (await operationFor(next.deploymentId))!;
expect(rebuilding).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: original.instanceId,
payload: { name: job.name, port: original.payload.port, nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
expect(rebuilding.id).not.toBe(original.id);
expect(nativeInstance(job.name)?.generation).toBe(1);
remote.state = 'running'; await acknowledge(next.deploymentId);
expect((await detail(next.deploymentId)).state).toBe('done');
expect(nativeInstance(job.name)?.generation).toBe(2);
// An interrupted original is reconciled to its already-failed receipt
// on the next worker pass; the failure itself is never cleared.
expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: 'Host runtime preflight failed' });
expect(await operationFor(job.deploymentId)).toMatchObject({ id: original.id, state: 'failed' });
expect(db().prepare("SELECT kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.rebuild','instance.restart') ORDER BY rowid").all())
.toEqual([{ kind: 'instance.create' }, { kind: 'instance.rebuild' }]);
});
it.each([
['create', false], ['create', true], ['rebuild', false], ['rebuild', true],
['recovery-restart', false], ['recovery-restart', true], ['recovery-rebuild', false], ['recovery-rebuild', true]
] as const)('resumes completed %s verification with the same command (receipt applied=%s)', async (action, applied) => {
let job = await queue(); await advance();
if (action === 'rebuild') {
await acknowledge(job.deploymentId);
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
const rebuild = queueNativeRebuild(job.name, 'test');
if (!rebuild.ok) throw new Error(rebuild.message);
job = rebuild; await advance();
} else if (action.startsWith('recovery')) {
remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
if (action === 'recovery-rebuild') await acknowledge(job.deploymentId);
}
const operation = (await operationFor(job.deploymentId))!;
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
remote.state = 'starting'; reportHostOperation(hostId, operation.id, { state: 'completed' });
if (applied) await advance();
const history = await import('../../src/lib/server/provisioning/history.js');
history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted for inspection' });
expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: true, canRetryWithRebuild: false });
const { db } = await import('../../src/lib/server/db/index.js');
const commands = db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all();
const { POST } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
expect(POST({ params: { id: job.deploymentId } }).status).toBe(202);
remote.state = 'running'; await advance();
expect((await detail(job.deploymentId)).state).toBe('done');
expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all()).toEqual(commands);
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
expect(nativeInstance(job.name)?.generation).toBe(operation.generation! + (operation.kind === 'instance.rebuild' ? 1 : 0));
expect(history.getDeployment(job.deploymentId)!.events).toContainEqual(expect.objectContaining({
kind: 'outcome', message: 'Failed: Verification interrupted for inspection'
}));
});
it.each(['pending', 'accepted', 'running', 'unknown', 'failed'] as const)('does not offer verification of a %s command or replay an ambiguous result', async state => {
const job = await queue(); await advance();
const operation = (await operationFor(job.deploymentId))!;
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
if (state !== 'pending') reportHostOperation(hostId, operation.id, { state });
const history = await import('../../src/lib/server/provisioning/history.js');
history.finishDeployment(job.deploymentId, { ok: false, message: 'Controller observation interrupted' });
expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: state === 'failed' });
expect(history.resumeVerification(job.deploymentId)).toBe(false);
if (state !== 'failed') {
const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js');
expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 });
}
const { db } = await import('../../src/lib/server/db/index.js');
if (state === 'unknown') db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId);
await advance();
expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").all())
.toEqual([{ id: operation.id }]);
});
it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work', 'creation cleanup', 'recovery attempt'])
('refuses to retry a failed creation after its %s changes', async change => {
const { job, original } = await failedCreation();
const { db } = await import('../../src/lib/server/db/index.js');
if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id);
if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id);
if (change === 'command host') {
const { createHost } = await import('../../src/lib/server/hosts/index.js');
db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id);
}
if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name);
if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
if (change === 'newer work') {
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
beginDeployment(job.name, 'restart');
}
if (change === 'creation cleanup') {
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
}
if (change === 'recovery attempt') db().prepare("UPDATE native_provisioning SET recovery_step='restart' WHERE deployment_id=?").run(job.deploymentId);
const history = await import('../../src/lib/server/provisioning/history.js');
const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js');
expect(history.getDeployment(job.deploymentId)?.canRetryWithRebuild).toBe(false);
expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 });
expect(db().prepare("SELECT id FROM server_deployments WHERE action='rebuild'").all()).toEqual([]);
});
it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work'])
('refuses to resume a completed creation after its %s changes', async change => {
const job = await queue(); await advance();
const original = (await operationFor(job.deploymentId))!;
const { reportHostOperation, createHost } = await import('../../src/lib/server/hosts/index.js');
reportHostOperation(hostId, original.id, { state: 'completed' });
const history = await import('../../src/lib/server/provisioning/history.js');
history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted' });
const { db } = await import('../../src/lib/server/db/index.js');
if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id);
if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id);
if (change === 'command host') db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id);
if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name);
if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
if (change === 'newer work') history.beginDeployment(job.name, 'restart');
expect(history.getDeployment(job.deploymentId)?.canResumeVerification).toBe(false);
expect(history.resumeVerification(job.deploymentId)).toBe(false);
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
});
});
describe('unhealthy recovery within native create and rebuild operations', () => {
async function launch(action: 'create' | 'rebuild') {
const job = await queue(); await advance();
tests/unit/provisioning-presentation.test.ts
+23
View File
@@ -19,6 +19,20 @@ function fixture(): ProvisioningDetail {
}
describe('provisioning outcome and evidence presentation', () => {
it('distinguishes a failed creation retry from readiness verification and its linked recovery', () => {
const detail = fixture();
detail.deployment.state = 'failed';
detail.canRetryWithRebuild = true;
expect(provisioningOutcome(detail)).toMatchObject({ active: false, headline: 'Create failed' });
expect(provisioningOutcome(detail).description).toContain('new attempt for the same lobby and port');
detail.canRetryWithRebuild = false;
detail.progress = progress({ recovery: { state: 'transferred', operationId: 'recovery-a', at, message: 'Rebuild requested' } });
expect(provisioningOutcome(detail).headline).toBe('Create failed');
expect(provisioningOutcome(detail).description).toContain('linked operation');
detail.progress = progress();
detail.canResumeVerification = true;
expect(provisioningOutcome(detail).description).toContain('remaining restart and rebuild recovery steps');
});
it('distinguishes cancellation from confirmed cleanup and validates its link data', () => {
const detail = fixture();
detail.deployment.state = 'cancelled';
@@ -142,6 +156,15 @@ describe('provisioning outcome and evidence presentation', () => {
});
describe('provisioning response and return-context boundaries', () => {
it('accepts optional retry eligibility only as an authoritative boolean', () => {
const detail = fixture();
for (const canRetryWithRebuild of [undefined, false, true]) {
expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(true);
}
for (const canRetryWithRebuild of ['true', 1, null]) {
expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(false);
}
});
it('accepts persisted usernames and legacy details but rejects malformed requesters', () => {
const detail = fixture();
for (const requested_by of [undefined, null, 'alice']) {
tests/unit/provisioning-refresh.test.ts
+9
View File
@@ -23,6 +23,15 @@ function deferred() {
}
describe('provisioning detail refresh ownership', () => {
it('requests a rebuild retry and refreshes the original failed operation', async () => {
const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true, deploymentId: 'recovery-a' })).mockResolvedValueOnce(response('a', 2));
const page = setup(fetcher);
page.setOperation('a', detail('a'));
await page.act('retry-rebuild', 'Follow the linked recovery operation.');
expect(fetcher).toHaveBeenNthCalledWith(1, '/api/provisioning/a/retry-rebuild', expect.objectContaining({ method: 'POST' }));
expect(fetcher).toHaveBeenNthCalledWith(2, '/api/provisioning/a?events=recent', expect.anything());
expect(page.state()).toMatchObject({ id: 'a', detail: detail('a', 2), notice: 'Follow the linked recovery operation.', busy: null });
});
it('sends explicit override confirmation and refreshes the operation after acceptance', async () => {
const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true })).mockResolvedValueOnce(response('a', 2));
const page = setup(fetcher);