342 lines
26 KiB
TypeScript
342 lines
26 KiB
TypeScript
import { putFileBlob } from '../../src/lib/server/db/file-blobs.js';
|
|
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
let fixture: Fixture;
|
|
const old = new Date(Date.now()-90*86400000).toISOString();
|
|
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
|
afterEach(async () => { await fixture.cleanup(); });
|
|
async function api() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
|
|
ensureHostSchema(db());
|
|
return { db: db(), ...await import('../../src/lib/server/retention.js'), ...await import('../../src/lib/server/settings.js') };
|
|
}
|
|
it('previews eligible campaigns, respects latest-N and explicit holds, and treats zero as indefinite', async () => {
|
|
const a = await api();
|
|
for (const id of ['a','b','c']) a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)").run(id,old,old);
|
|
a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 1,historyKeepCount: 1 });
|
|
a.setRetentionControl({ category: 'campaign',id: 'a',protected: true });
|
|
expect(a.retentionPreview().find(p=>p.category==='campaign')?.eligible).toBe(1);
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([{ id:'a' },{ id:'c' }]);
|
|
a.setRetentionControl({ category: 'campaign',id: 'a',protected: false });
|
|
a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 0,historyKeepCount: 0 });
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
});
|
|
it('seeks expired campaigns and uses indexed dependency checks even with failure retention enabled', async () => {
|
|
const a = await api();
|
|
const now = new Date().toISOString();
|
|
const insert = a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)");
|
|
a.db.transaction(() => {
|
|
for (let n=0;n<8000;n++) insert.run(`recent-${n}`,now,now);
|
|
})();
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:7,failureKeepDays:7});
|
|
const prepare = vi.spyOn(a.db,'prepare');
|
|
let sql: string;
|
|
try {
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
sql = prepare.mock.calls.find(([sql]) => sql.startsWith('SELECT o.rowid AS rid'))![0];
|
|
} finally { prepare.mockRestore(); }
|
|
const plan = a.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all({category:'campaign',cutoff:old,failureCutoff:old}) as {detail:string}[];
|
|
const details = plan.map(row=>row.detail).join('\n');
|
|
expect(details).toMatch(/SEARCH o USING (?:COVERING )?INDEX idx_deployments_started \(started_at<\?\)/);
|
|
for (const index of ['idx_deployments_rolled_back_by','idx_watchdog_actions_campaign','idx_provisioning_sources_source']) {
|
|
expect(details).toContain(index);
|
|
}
|
|
expect(details).not.toMatch(/SCAN [odas]\b/);
|
|
});
|
|
it('selects resolved failures separately while preserving age, protection and batch limits', async () => {
|
|
const a = await api();
|
|
const now = new Date().toISOString();
|
|
for (const id of ['normal','unresolved','resolved','held','new-resolution','changed']) {
|
|
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)").run(id,old,old);
|
|
if (id !== 'normal') a.db.prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES(?,'s',1,'failed')").run(id);
|
|
}
|
|
for (const id of ['resolved','held','new-resolution','changed']) {
|
|
a.setRetentionControl({category:'campaign',id,resolve:true,protected:id==='held'});
|
|
if (id !== 'new-resolution') a.db.prepare("UPDATE retention_controls SET resolved_at=? WHERE record_id=?").run(old,id);
|
|
}
|
|
a.db.prepare("UPDATE deployments SET started_at=? WHERE id='changed'").run(now);
|
|
// Failure retention works independently of normal campaign retention.
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:0,failureKeepDays:7});
|
|
expect(a.retentionPreview().find(p=>p.category==='campaign')?.records).toEqual([{id:'resolved'}]);
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:7});
|
|
const insert = a.db.prepare("INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)");
|
|
a.db.transaction(() => { for (let n=0;n<600;n++) insert.run(`old-${n}`,old); })();
|
|
expect(a.pruneHistory('campaign')).toBe(500);
|
|
expect(a.pruneHistory('campaign')).toBe(101);
|
|
expect(a.db.prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([
|
|
{id:'changed'},{id:'held'},{id:'new-resolution'},{id:'unresolved'}
|
|
]);
|
|
});
|
|
it('expires file bodies without silently turning rollback into file deletion', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('p','promote',?,?)").run(old,old);
|
|
a.db.prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before_hash,content_after_hash) VALUES('p','s','updated','test.cfg',?,?)").run(putFileBlob(Buffer.from('before'), a.db),putFileBlob(Buffer.from('after'), a.db));
|
|
a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 0,fileContentKeepDays: 1 });
|
|
expect(a.retentionPreview().find(p=>p.category==='file-content')?.payloadBytes).toBe(11);
|
|
expect(a.pruneHistory('file-content')).toBe(1);
|
|
expect(a.db.prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,(SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files").get()).toEqual({ content_before:null,content_after:null });
|
|
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
|
|
await expect(rollbackPromoteDeployment('p')).rejects.toThrow('expired');
|
|
expect(a.pruneHistory('file-content')).toBe(0);
|
|
});
|
|
it('requires explicit failure resolution and preserves recovery slots even after resolution', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO servers(name) VALUES('s')").run();
|
|
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('failure','s','restart','failed','failed',?,?,'test')").run(old,old);
|
|
a.db.prepare("INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES('s','failure',?)").run(old);
|
|
a.saveFleetSettings({ ...a.fleetSettings(),provisioningKeepDays: 1,failureKeepDays: 1 });
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.setRetentionControl({ category:'provisioning',id:'failure',resolve:true });
|
|
a.db.prepare("UPDATE retention_controls SET resolved_at=?").run(old);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.db.prepare("UPDATE restart_rollout SET healthy_at=?").run(new Date().toISOString());
|
|
expect(a.pruneHistory('provisioning')).toBe(1);
|
|
});
|
|
it('compacts action history but preserves idempotency and terminal receipts', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name:'Host' });
|
|
const op = hosts.queueHostOperation(host.id,'files.stage',{ file:'large' },{ idempotencyKey:'original' });
|
|
hosts.reportHostOperation(host.id,op.id,{ state:'completed',result:{ done:true } });
|
|
a.db.prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?').run(old,op.id);
|
|
a.saveFleetSettings({ ...a.fleetSettings(),actionKeepDays:1 });
|
|
expect(a.pruneHistory('operation')).toBe(1);
|
|
expect(hosts.listHostOperations(host.id)).toHaveLength(0);
|
|
expect(hosts.queueHostOperation(host.id,'files.stage',{ file:'large' },{ idempotencyKey:'original' }).id).toBe(op.id);
|
|
expect(()=>hosts.queueHostOperation(host.id,'files.stage',{ file:'different' },{ idempotencyKey:'original' })).toThrow('different request');
|
|
expect(hosts.pendingHostOperations(host.id)).toHaveLength(0);
|
|
expect(hosts.reportHostOperation(host.id,op.id,{state:'running'}).state).toBe('completed');
|
|
});
|
|
it('deletes old maintenance jobs while preserving update suppression evidence and protected jobs', async () => {
|
|
const a = await api();
|
|
for (const kind of ['legacy','schedule','update']) a.db.prepare("INSERT INTO maintenance_jobs(server,state,kind,requested_at,finished_at) VALUES(?,'done',?,?,?)").run(kind,kind,old,old);
|
|
a.setRetentionControl({category:'maintenance',id:'1',protected:true});
|
|
expect(a.pruneHistory('maintenance')).toBe(1);
|
|
expect(a.db.prepare('SELECT kind FROM maintenance_jobs ORDER BY id').all()).toEqual([{kind:'legacy'},{kind:'update'}]);
|
|
});
|
|
it('keeps current and referenced artifact versions and honors the minimum version count', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO scripts(id,name,target_rel_path,current_version_id) VALUES(1,'test','test.gsc',1)").run();
|
|
for(let n=1;n<=4;n++) a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash,uploaded_at) VALUES(?,1,?,'sha',1,?,?)").run(n,n,putFileBlob(Buffer.from([1]), a.db),old);
|
|
a.saveFleetSettings({...a.fleetSettings(),versionKeepCount:2});
|
|
expect(a.pruneHistory('version')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM script_versions ORDER BY id').all()).toEqual([{id:1},{id:3},{id:4}]);
|
|
});
|
|
it('prunes metrics and diagnostics while leaving unlimited histories intact', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,'s','t5','kino',0,4)").run(Date.parse(old));
|
|
a.saveFleetSettings({...a.fleetSettings(),metricsKeepDays:0});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({n:1});
|
|
a.saveFleetSettings({...a.fleetSettings(),metricsKeepDays:1});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({n:0});
|
|
});
|
|
|
|
it('prunes closed matches and audit events without deleting active matches or protected audit records', async () => {
|
|
const a = await api();
|
|
for (const status of ['active','completed','interrupted']) a.db.prepare(`INSERT INTO leaderboard_matches(server,game,map,map_key,map_name,started_at,last_received_at,status,last_event,last_fingerprint)
|
|
VALUES('s','t5','kino','kino','Kino',?,?,?,'game_end','test')`).run(old,old,status);
|
|
// Preview initializes the optional authentication schema.
|
|
a.retentionPreview();
|
|
a.db.prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)").run(Date.parse(old));
|
|
a.db.prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)").run(Date.parse(old));
|
|
a.setRetentionControl({category:'audit',id:'1',protected:true});
|
|
a.saveFleetSettings({...a.fleetSettings(),matchKeepDays:1,auditKeepDays:1});
|
|
expect(a.pruneHistory('match')).toBe(2);
|
|
expect(a.pruneHistory('audit')).toBe(1);
|
|
expect(a.db.prepare('SELECT status FROM leaderboard_matches').all()).toEqual([{status:'active'}]);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
it('keeps active beta evidence while expiring closed snapshots and then closed history', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'s','s.gsc')").run();
|
|
a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash) VALUES(1,1,1,'sha',3,?)").run(putFileBlob(Buffer.from([1,2,3]), a.db));
|
|
for (const [id,state] of [[1,'running'],[2,'rolled-back']] as const) {
|
|
a.db.prepare("INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at,closed_at) VALUES(?,1,1,'test',?,?,?)").run(id,state,old,old);
|
|
a.db.prepare("INSERT INTO beta_revisions(beta_id,revision,started_at) VALUES(?,1,?)").run(id,old);
|
|
a.db.prepare("INSERT INTO beta_revision_files(beta_id,revision,script_id,name,kind,path,version_id,version,sha256,content_hash) VALUES(?,1,1,'s','script','s.gsc',1,1,'sha',?)").run(id,putFileBlob(Buffer.from([1,2,3]), a.db));
|
|
}
|
|
a.saveFleetSettings({...a.fleetSettings(),fileContentKeepDays:1});
|
|
expect(a.pruneHistory('beta-file-content')).toBe(1);
|
|
expect(a.db.prepare('SELECT beta_id,COALESCE((SELECT size FROM file_blobs WHERE sha256=content_hash),0) AS bytes FROM beta_revision_files ORDER BY beta_id').all()).toEqual([{beta_id:1,bytes:3},{beta_id:2,bytes:0}]);
|
|
a.saveFleetSettings({...a.fleetSettings(),betaKeepDays:1});
|
|
expect(a.pruneHistory('beta')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM betas').all()).toEqual([{id:1}]);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
it('applies minimum count and holds to routine cleanup and its preview consistently', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({name:'Host'});
|
|
for (let n=1;n<=3;n++) a.db.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,observation_success,routine) VALUES(?,?,'files.read','{}','completed',?,?,1,1)").run(String(n),host.id,old,old);
|
|
a.setRetentionControl({category:'observation',id:'1',protected:true});
|
|
a.saveFleetSettings({...a.fleetSettings(),historyKeepCount:1});
|
|
expect(a.retentionPreview().find(p=>p.category==='observation')?.eligible).toBe(1);
|
|
expect(a.pruneHistory('observation')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM ec_host_operations ORDER BY id').all()).toEqual([{id:'1'},{id:'3'}]);
|
|
});
|
|
it('never removes the calibration window when choosing a shorter resource retention policy', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({name:'Host'});
|
|
a.db.prepare('INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,?)').run(host.id,old);
|
|
a.db.prepare("INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,'i',1,'startup',?,1,1024)").run(host.id,old);
|
|
a.saveFleetSettings({...a.fleetSettings(),hostResourceKeepDays:0,instanceResourceKeepDays:0});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({n:1});
|
|
a.saveFleetSettings({...a.fleetSettings(),hostResourceKeepDays:1,instanceResourceKeepDays:1});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({n:0});
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM ec_host_resource_samples').get()).toEqual({n:0});
|
|
});
|
|
it('previews draft retention without saving it and restricts retention tools to administrators', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('preview','drift',?,?)").run(old,old);
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:0});
|
|
const {POST} = await import('../../src/routes/api/settings/retention/preview/+server.js');
|
|
const response = await POST({request:new Request('http://localhost/api/settings/retention/preview',{method:'POST',body:JSON.stringify({...a.fleetSettings(),deployKeepDays:1})})});
|
|
expect(response.status).toBe(200);
|
|
expect((await response.json()).policies.find((p:{category:string})=>p.category==='campaign').eligible).toBe(1);
|
|
expect(a.fleetSettings().deployKeepDays).toBe(0);
|
|
expect(a.db.prepare('SELECT id FROM deployments').get()).toEqual({id:'preview'});
|
|
const {routePermission} = await import('../../src/lib/server/auth/access.js');
|
|
expect(routePermission('/api/settings/retention','GET')).toBe('admin');
|
|
expect(routePermission('/api/settings/retention/preview','POST')).toBe('admin');
|
|
});
|
|
it('protects cancelled workflows and linked file history through ambiguous and unreconciled receipts', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host=hosts.createHost({name:'Host'});
|
|
a.db.prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)").run(host.id,old);
|
|
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','restart','cancelled','cancelled',?,?,'test')").run(old,old);
|
|
const command=hosts.queueHostOperation(host.id,'instance.restart',{stageId:'campaign'});
|
|
a.db.prepare("UPDATE ec_host_operations SET state='unknown' WHERE id=?").run(command.id);
|
|
a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,files_json) VALUES('cancelled','instance',1,?,'[{\"id\":1}]')").run(command.id);
|
|
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)").run(old,old);
|
|
a.db.prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')").run();
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:1,provisioningKeepDays:1,fileContentKeepDays:1});
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
expect(a.pruneHistory('file-content')).toBe(0);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.db.prepare("UPDATE ec_host_operations SET state='completed' WHERE id=?").run(command.id);
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.db.prepare("UPDATE native_provisioning SET applied_at=? WHERE deployment_id='cancelled'").run(new Date().toISOString());
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
expect(a.pruneHistory('provisioning')).toBe(1);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
|
|
it('retains a cancelled rebuild and its source until the partial-failure receipt is applied', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Host' });
|
|
a.db.prepare("INSERT INTO servers(name,container_state) VALUES('s','running')").run();
|
|
a.db.prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)").run(host.id, old);
|
|
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')").run(old, old);
|
|
const command = hosts.queueHostOperation(host.id, 'instance.rebuild', { nextGeneration: 2 }, { instanceId: 'instance', generation: 1 });
|
|
a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('cancelled','instance',1,?)").run(command.id);
|
|
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)").run(old, old);
|
|
a.db.prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')").run();
|
|
hosts.reportHostOperation(host.id, command.id, { state: 'failed', error: 'Replacement creation failed', result: {
|
|
reconciled: true, rebuildFailure: { phase: 'disrupting', generation: 1, nextGeneration: 2,
|
|
oldContainerMissing: true, replacementContainerMissing: true, replacementFilesRemoved: true }
|
|
} });
|
|
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 1, provisioningKeepDays: 1, historyKeepCount: 0 });
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
expect(a.db.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'").get()).toEqual({ applied_at: null });
|
|
|
|
const { reconcileNativeProvisioningReceipt } = await import('../../src/lib/server/provisioning/native.js');
|
|
reconcileNativeProvisioningReceipt('cancelled');
|
|
expect(a.db.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'").get()).toEqual({ applied_at: expect.any(String) });
|
|
expect(a.db.prepare("SELECT container_state FROM servers WHERE name='s'").get()).toEqual({ container_state: 'missing' });
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
expect(a.pruneHistory('provisioning')).toBe(1);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
|
|
async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Recovery host' });
|
|
const { config } = await import('../../src/lib/server/config.js');
|
|
hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1, dockerRunning: true,
|
|
directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [config.defaultRuntimeImage],
|
|
libraries: [{ id: 'lib', state: 'ready' }] });
|
|
const generation = action === 'create' ? 1 : 2;
|
|
a.db.prepare("INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,?,'lib','image',1,?)")
|
|
.run(host.id, generation, old);
|
|
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('recovering','s',?,'awaiting-health','Recovery',?,'test')")
|
|
.run(action === 'create' ? 'create' : 'rebuild', old);
|
|
const sha256 = putFileBlob(Buffer.from('original launch files'), a.db);
|
|
const launch = hosts.queueHostOperation(host.id, action === 'rebuild' ? 'instance.rebuild' : 'instance.create', {
|
|
name: 's', files: [{ path: 'configs/server.cfg', sha256, contentSha256: sha256 }]
|
|
}, { instanceId: 'instance', generation: action === 'rebuild' ? 1 : generation,
|
|
idempotencyKey: action === 'migration' ? 'recovering:migration-create:0' : 'recovering' });
|
|
hosts.reportHostOperation(host.id, launch.id, { state: 'completed' });
|
|
const restart = hosts.queueHostOperation(host.id, 'instance.restart', { unhealthyRecovery: true }, { instanceId: 'instance', generation });
|
|
a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,recovery_step) VALUES('recovering','instance',?,?,'restart')")
|
|
.run(generation, restart.id);
|
|
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
|
|
a.saveFleetSettings({ ...a.fleetSettings(), actionKeepDays: 1, payloadKeepDays: 1, historyKeepCount: 0 });
|
|
return { ...a, hosts, host, generation, launch, restart, sha256 };
|
|
}
|
|
|
|
it.each(['create', 'rebuild', 'migration'] as const)('keeps original %s launch files and ownership through recovery and releases them after success', async action => {
|
|
const a = await recoveryLaunch(action);
|
|
expect(a.pruneHistory('operation-payload')).toBe(0);
|
|
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([
|
|
{ path: 'configs/server.cfg', sha256: a.sha256, contentBase64: Buffer.from('original launch files').toString('base64') }
|
|
]);
|
|
a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' });
|
|
const rebuild = a.hosts.queueHostOperation(a.host.id, 'instance.rebuild', { unhealthyRecovery: true, nextGeneration: a.generation + 1 },
|
|
{ instanceId: 'instance', generation: a.generation });
|
|
a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' });
|
|
a.db.prepare("UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'")
|
|
.run(rebuild.id, old);
|
|
a.db.prepare("UPDATE native_instances SET generation=generation+1 WHERE id='instance'").run();
|
|
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
|
|
a.pruneHistory('operation-payload');
|
|
a.pruneHistory('operation');
|
|
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
|
|
.toEqual({ history_expired_at: null });
|
|
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([
|
|
{ path: 'configs/server.cfg', sha256: a.sha256, contentBase64: Buffer.from('original launch files').toString('base64') }
|
|
]);
|
|
a.db.prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id='recovering'").run(old);
|
|
a.pruneHistory('operation-payload');
|
|
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([{ path: 'configs/server.cfg', sha256: a.sha256 }]);
|
|
a.pruneHistory('operation');
|
|
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
|
|
.toEqual({ history_expired_at: expect.any(String) });
|
|
});
|
|
|
|
it.each(['cancelled', 'failed'])('keeps original ownership for a %s recovered creation until a later lifecycle operation supersedes cleanup', async state => {
|
|
const a = await recoveryLaunch('create');
|
|
a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' });
|
|
const rebuild = a.hosts.queueHostOperation(a.host.id, 'instance.rebuild', { unhealthyRecovery: true, nextGeneration: 2 },
|
|
{ instanceId: 'instance', generation: 1 });
|
|
a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' });
|
|
a.db.prepare("UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'")
|
|
.run(rebuild.id, old);
|
|
a.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run();
|
|
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
|
|
a.db.prepare("UPDATE server_deployments SET state=?,finished_at=? WHERE id='recovering'").run(state, old);
|
|
const { canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(canCancelCreation('recovering')).toBe(true);
|
|
a.pruneHistory('operation');
|
|
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
|
|
.toEqual({ history_expired_at: null });
|
|
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('replacement','s','restart','done','Done',?,?,'test')")
|
|
.run(old, old);
|
|
expect(canCancelCreation('recovering')).toBe(false);
|
|
a.pruneHistory('operation');
|
|
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
|
|
.toEqual({ history_expired_at: expect.any(String) });
|
|
});
|