Reconcile rebuilds interrupted before replacement creation

This commit is contained in:
2026-10-08 14:24:52 -04:00
parent e01b143a09
commit c4e2ff4384
11 changed files with 571 additions and 14 deletions
+29
View File
@@ -231,6 +231,35 @@ it('protects cancelled workflows and linked file history through ambiguous and u
expect(a.db.pragma('foreign_key_check')).toEqual([]);
});
it('retains a cancelled rebuild and its source until the partial-failure receipt is applied', async () => {
const a = await api();
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Host' });
a.db.prepare("INSERT INTO servers(name,container_state) VALUES('s','running')").run();
a.db.prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)").run(host.id, old);
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')").run(old, old);
const command = hosts.queueHostOperation(host.id, 'instance.rebuild', { nextGeneration: 2 }, { instanceId: 'instance', generation: 1 });
a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('cancelled','instance',1,?)").run(command.id);
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)").run(old, old);
a.db.prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')").run();
hosts.reportHostOperation(host.id, command.id, { state: 'failed', error: 'Replacement creation failed', result: {
reconciled: true, rebuildFailure: { phase: 'disrupting', generation: 1, nextGeneration: 2,
oldContainerMissing: true, replacementContainerMissing: true, replacementFilesRemoved: true }
} });
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 1, provisioningKeepDays: 1, historyKeepCount: 0 });
expect(a.pruneHistory('campaign')).toBe(0);
expect(a.pruneHistory('provisioning')).toBe(0);
expect(a.db.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'").get()).toEqual({ applied_at: null });
const { reconcileNativeProvisioningReceipt } = await import('../../src/lib/server/provisioning/native.js');
reconcileNativeProvisioningReceipt('cancelled');
expect(a.db.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'").get()).toEqual({ applied_at: expect.any(String) });
expect(a.db.prepare("SELECT container_state FROM servers WHERE name='s'").get()).toEqual({ container_state: 'missing' });
expect(a.pruneHistory('campaign')).toBe(1);
expect(a.pruneHistory('provisioning')).toBe(1);
expect(a.db.pragma('foreign_key_check')).toEqual([]);
});
async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
const a = await api();
const hosts = await import('../../src/lib/server/hosts/index.js');