josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Reconcile rebuilds interrupted before replacement creation

Browse Source
This commit is contained in:
josh
2026-10-08 14:24:52 -04:00
parent e01b143a09
commit c4e2ff4384
11 changed files with 571 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/native-provisioning.test.ts
+135
View File
@@ -1946,6 +1946,141 @@ describe('native provisioning cancellation and rollback', () => {
});
});
describe('partial rebuild failure reconciliation', () => {
const originalError = 'Replacement game library could not be prepared';
const failure = (generation = 1) => ({ reconciled: true, originalError, rebuildFailure: {
phase: 'disrupting', generation, nextGeneration: generation + 1,
oldContainerMissing: true, replacementContainerMissing: true, replacementFilesRemoved: true
} });
async function cancelledRebuild() {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
const first = requestLifecycle(job.name, 'rebuild');
if (!first.ok) throw new Error(first.error);
await advance();
const command = (await operationFor(first.deploymentId))!;
const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
pendingHostOperations(hostId);
reportHostOperation(hostId, command.id, { state: 'unknown', error: originalError });
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
expect(cancelProvisioningJob(first.deploymentId)).toBe(true);
const replacement = requestLifecycle(job.name, 'rebuild');
if (!replacement.ok) throw new Error(replacement.error);
return { job, first, command, replacement };
}
it('records missing containers before releasing a queued rebuild without replaying the cancelled command', async () => {
const { job, first, command, replacement } = await cancelledRebuild();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
const { getHostOperation, reportHostOperation, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
const { nativeInstance, nativeRequest } = await import('../../src/lib/server/backends/native.js');
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { unsettledCancelledCommand } = await import('../../src/lib/server/provisioning/cancellation.js');
const ports = db().prepare('SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol').all();
await advance();
expect(await operationFor(replacement.deploymentId)).toBeNull();
expect(getDeployment(replacement.deploymentId)?.progress?.blocker).toMatchObject({ code: 'cancelled-command',
href: `/provisioning/${first.deploymentId}`, label: 'View cancelled operation',
message: expect.stringContaining(originalError) });
expect(getDeployment(replacement.deploymentId)?.progress?.blocker?.message).toContain('unknown');
reportHostOperation(hostId, command.id, { state: 'failed',
error: `Old container removed; replacement creation failed. ${originalError}`, result: failure() });
expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).not.toBeNull();
expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
.toEqual({ applied_at: null });
remote.state = 'missing';
vi.mocked(nativeRequest).mockImplementationOnce(async () => {
expect(db().prepare('SELECT container_state,last_players FROM servers WHERE name=?').get(job.name))
.toEqual({ container_state: 'missing', last_players: null });
expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
.toEqual({ applied_at: expect.any(String) });
expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).toBeNull();
return { exists: false, healthy: false, state: 'missing' } as never;
});
closeDb(); await advance();
const next = (await operationFor(replacement.deploymentId))!;
expect(next).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: command.instanceId,
payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
expect(next.id).not.toBe(command.id);
expect(nativeInstance(job.name)).toMatchObject({ id: command.instanceId, generation: 1, retired_at: null });
expect(getServerKey(job.name)).toBe('saved-key');
expect(db().prepare('SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol').all()).toEqual(ports);
expect((await detail(first.deploymentId)).state).toBe('cancelled');
expect(getHostOperation(command.id)).toMatchObject({ state: 'failed', result: failure() });
expect(pendingHostOperations(hostId).some(operation => operation.id === command.id)).toBe(false);
await advance();
expect((await operationFor(replacement.deploymentId))!.id).toBe(next.id);
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 2 });
remote.state = 'running'; await acknowledge(replacement.deploymentId);
expect((await detail(replacement.deploymentId)).state).toBe('done');
expect(nativeInstance(job.name)?.generation).toBe(2);
expect((await detail(first.deploymentId)).state).toBe('cancelled');
});
it.each(['disrupting', 'creating'])('keeps an ambiguous %s rebuild blocked even when inspections currently see no containers', async phase => {
const { job, first, command, replacement } = await cancelledRebuild();
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
reportHostOperation(hostId, command.id, { state: 'unknown', error: originalError,
result: { checkpoint: { phase }, state: 'missing', oldContainerMissing: true, replacementContainerMissing: true } });
remote.state = 'missing';
const { nativeRequest, nativeInstance } = await import('../../src/lib/server/backends/native.js');
vi.mocked(nativeRequest).mockClear();
const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb();
await advance(); await advance();
expect(await operationFor(replacement.deploymentId)).toBeNull();
expect(vi.mocked(nativeRequest)).not.toHaveBeenCalled();
expect((await operationFor(first.deploymentId))!.id).toBe(command.id);
expect(nativeInstance(job.name)?.generation).toBe(1);
expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
.toEqual({ applied_at: null });
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
});
it.each(['creation started', 'replacement files remain', 'wrong target generation', 'command target mismatch'])('does not release a replacement from invalid partial evidence: %s', async evidence => {
const { first, command, replacement } = await cancelledRebuild();
const result = failure();
if (evidence === 'creation started') result.rebuildFailure.phase = 'creating';
if (evidence === 'replacement files remain') result.rebuildFailure.replacementFilesRemoved = false;
if (evidence === 'wrong target generation') result.rebuildFailure.nextGeneration = 3;
const { db } = await import('../../src/lib/server/db/index.js');
if (evidence === 'command target mismatch') db().prepare("UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?")
.run(command.id);
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
reportHostOperation(hostId, command.id, { state: 'failed', error: originalError, result });
remote.state = 'missing'; await advance();
expect(await operationFor(replacement.deploymentId)).toBeNull();
expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
.toEqual({ applied_at: null });
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
});
it('records an uncancelled partial rebuild failure without claiming replacement generation or successful health', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
const first = requestLifecycle(job.name, 'rebuild');
if (!first.ok) throw new Error(first.error);
await advance();
const command = (await operationFor(first.deploymentId))!;
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
reportHostOperation(hostId, command.id, { state: 'failed', result: failure(),
error: `Old container removed; replacement creation failed. ${originalError}` });
await advance();
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect((await detail(first.deploymentId)).state).toBe('failed');
expect(nativeInstance(job.name)).toMatchObject({ generation: 1, retired_at: null });
expect(db().prepare('SELECT container_state,last_players FROM servers WHERE name=?').get(job.name))
.toEqual({ container_state: 'missing', last_players: null });
expect(db().prepare('SELECT applied_at,verified_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
.toEqual({ applied_at: expect.any(String), verified_at: null });
expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE instance_id=?').get(command.instanceId)).toEqual({ count: 2 });
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
expect(getServerKey(job.name)).toBe('saved-key');
});
});
describe('cancelled native mutation bookkeeping', () => {
it.each([null, 'before beta'])('keeps the original host receipt across beta revisions (original=%s)', async original => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
tests/unit/retention.test.ts
+29
View File
@@ -231,6 +231,35 @@ it('protects cancelled workflows and linked file history through ambiguous and u
expect(a.db.pragma('foreign_key_check')).toEqual([]);
});
it('retains a cancelled rebuild and its source until the partial-failure receipt is applied', async () => {
const a = await api();
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Host' });
a.db.prepare("INSERT INTO servers(name,container_state) VALUES('s','running')").run();
a.db.prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)").run(host.id, old);
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')").run(old, old);
const command = hosts.queueHostOperation(host.id, 'instance.rebuild', { nextGeneration: 2 }, { instanceId: 'instance', generation: 1 });
a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('cancelled','instance',1,?)").run(command.id);
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)").run(old, old);
a.db.prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')").run();
hosts.reportHostOperation(host.id, command.id, { state: 'failed', error: 'Replacement creation failed', result: {
reconciled: true, rebuildFailure: { phase: 'disrupting', generation: 1, nextGeneration: 2,
oldContainerMissing: true, replacementContainerMissing: true, replacementFilesRemoved: true }
} });
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 1, provisioningKeepDays: 1, historyKeepCount: 0 });
expect(a.pruneHistory('campaign')).toBe(0);
expect(a.pruneHistory('provisioning')).toBe(0);
expect(a.db.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'").get()).toEqual({ applied_at: null });
const { reconcileNativeProvisioningReceipt } = await import('../../src/lib/server/provisioning/native.js');
reconcileNativeProvisioningReceipt('cancelled');
expect(a.db.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'").get()).toEqual({ applied_at: expect.any(String) });
expect(a.db.prepare("SELECT container_state FROM servers WHERE name='s'").get()).toEqual({ container_state: 'missing' });
expect(a.pruneHistory('campaign')).toBe(1);
expect(a.pruneHistory('provisioning')).toBe(1);
expect(a.db.pragma('foreign_key_check')).toEqual([]);
});
async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
const a = await api();
const hosts = await import('../../src/lib/server/hosts/index.js');