josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Recover unhealthy containers during create and rebuild

Browse Source
This commit is contained in:
josh
2026-10-08 13:08:33 -04:00
parent 012cd51a66
commit 368e457dc6
18 changed files with 881 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/retention.test.ts
+80
View File
@@ -230,3 +230,83 @@ it('protects cancelled workflows and linked file history through ambiguous and u
expect(a.pruneHistory('provisioning')).toBe(1);
expect(a.db.pragma('foreign_key_check')).toEqual([]);
});
async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
const a = await api();
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Recovery host' });
const { config } = await import('../../src/lib/server/config.js');
hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1, dockerRunning: true,
directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [config.defaultRuntimeImage],
libraries: [{ id: 'lib', state: 'ready' }] });
const generation = action === 'create' ? 1 : 2;
a.db.prepare("INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,?,'lib','image',1,?)")
.run(host.id, generation, old);
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('recovering','s',?,'awaiting-health','Recovery',?,'test')")
.run(action === 'create' ? 'create' : 'rebuild', old);
const sha256 = putFileBlob(Buffer.from('original launch files'), a.db);
const launch = hosts.queueHostOperation(host.id, action === 'rebuild' ? 'instance.rebuild' : 'instance.create', {
name: 's', files: [{ path: 'configs/server.cfg', sha256, contentSha256: sha256 }]
}, { instanceId: 'instance', generation: action === 'rebuild' ? 1 : generation,
idempotencyKey: action === 'migration' ? 'recovering:migration-create:0' : 'recovering' });
hosts.reportHostOperation(host.id, launch.id, { state: 'completed' });
const restart = hosts.queueHostOperation(host.id, 'instance.restart', { unhealthyRecovery: true }, { instanceId: 'instance', generation });
a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,recovery_step) VALUES('recovering','instance',?,?,'restart')")
.run(generation, restart.id);
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
a.saveFleetSettings({ ...a.fleetSettings(), actionKeepDays: 1, payloadKeepDays: 1, historyKeepCount: 0 });
return { ...a, hosts, host, generation, launch, restart, sha256 };
}
it.each(['create', 'rebuild', 'migration'] as const)('keeps original %s launch files and ownership through recovery and releases them after success', async action => {
const a = await recoveryLaunch(action);
expect(a.pruneHistory('operation-payload')).toBe(0);
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([
{ path: 'configs/server.cfg', sha256: a.sha256, contentBase64: Buffer.from('original launch files').toString('base64') }
]);
a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' });
const rebuild = a.hosts.queueHostOperation(a.host.id, 'instance.rebuild', { unhealthyRecovery: true, nextGeneration: a.generation + 1 },
{ instanceId: 'instance', generation: a.generation });
a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' });
a.db.prepare("UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'")
.run(rebuild.id, old);
a.db.prepare("UPDATE native_instances SET generation=generation+1 WHERE id='instance'").run();
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
a.pruneHistory('operation-payload');
a.pruneHistory('operation');
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
.toEqual({ history_expired_at: null });
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([
{ path: 'configs/server.cfg', sha256: a.sha256, contentBase64: Buffer.from('original launch files').toString('base64') }
]);
a.db.prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id='recovering'").run(old);
a.pruneHistory('operation-payload');
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([{ path: 'configs/server.cfg', sha256: a.sha256 }]);
a.pruneHistory('operation');
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
.toEqual({ history_expired_at: expect.any(String) });
});
it.each(['cancelled', 'failed'])('keeps original ownership for a %s recovered creation until a later lifecycle operation supersedes cleanup', async state => {
const a = await recoveryLaunch('create');
a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' });
const rebuild = a.hosts.queueHostOperation(a.host.id, 'instance.rebuild', { unhealthyRecovery: true, nextGeneration: 2 },
{ instanceId: 'instance', generation: 1 });
a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' });
a.db.prepare("UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'")
.run(rebuild.id, old);
a.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run();
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
a.db.prepare("UPDATE server_deployments SET state=?,finished_at=? WHERE id='recovering'").run(state, old);
const { canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
expect(canCancelCreation('recovering')).toBe(true);
a.pruneHistory('operation');
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
.toEqual({ history_expired_at: null });
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('replacement','s','restart','done','Done',?,?,'test')")
.run(old, old);
expect(canCancelCreation('recovering')).toBe(false);
a.pruneHistory('operation');
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
.toEqual({ history_expired_at: expect.any(String) });
});