232 lines
17 KiB
TypeScript
232 lines
17 KiB
TypeScript
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
let fixture: Fixture;
|
|
const old = new Date(Date.now()-90*86400000).toISOString();
|
|
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
|
afterEach(async () => { await fixture.cleanup(); });
|
|
async function api() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
|
|
ensureHostSchema(db());
|
|
return { db: db(), ...await import('../../src/lib/server/retention.js'), ...await import('../../src/lib/server/settings.js') };
|
|
}
|
|
it('previews eligible campaigns, respects latest-N and explicit holds, and treats zero as indefinite', async () => {
|
|
const a = await api();
|
|
for (const id of ['a','b','c']) a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)").run(id,old,old);
|
|
a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 1,historyKeepCount: 1 });
|
|
a.setRetentionControl({ category: 'campaign',id: 'a',protected: true });
|
|
expect(a.retentionPreview().find(p=>p.category==='campaign')?.eligible).toBe(1);
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([{ id:'a' },{ id:'c' }]);
|
|
a.setRetentionControl({ category: 'campaign',id: 'a',protected: false });
|
|
a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 0,historyKeepCount: 0 });
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
});
|
|
it('seeks expired campaigns and uses indexed dependency checks even with failure retention enabled', async () => {
|
|
const a = await api();
|
|
const now = new Date().toISOString();
|
|
const insert = a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)");
|
|
a.db.transaction(() => {
|
|
for (let n=0;n<8000;n++) insert.run(`recent-${n}`,now,now);
|
|
})();
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:7,failureKeepDays:7});
|
|
const prepare = vi.spyOn(a.db,'prepare');
|
|
let sql: string;
|
|
try {
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
sql = prepare.mock.calls.find(([sql]) => sql.startsWith('SELECT o.rowid AS rid'))![0];
|
|
} finally { prepare.mockRestore(); }
|
|
const plan = a.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all({category:'campaign',cutoff:old,failureCutoff:old}) as {detail:string}[];
|
|
const details = plan.map(row=>row.detail).join('\n');
|
|
expect(details).toMatch(/SEARCH o USING (?:COVERING )?INDEX idx_deployments_started \(started_at<\?\)/);
|
|
for (const index of ['idx_deployments_rolled_back_by','idx_watchdog_actions_campaign','idx_provisioning_sources_source']) {
|
|
expect(details).toContain(index);
|
|
}
|
|
expect(details).not.toMatch(/SCAN [odas]\b/);
|
|
});
|
|
it('selects resolved failures separately while preserving age, protection and batch limits', async () => {
|
|
const a = await api();
|
|
const now = new Date().toISOString();
|
|
for (const id of ['normal','unresolved','resolved','held','new-resolution','changed']) {
|
|
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)").run(id,old,old);
|
|
if (id !== 'normal') a.db.prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES(?,'s',1,'failed')").run(id);
|
|
}
|
|
for (const id of ['resolved','held','new-resolution','changed']) {
|
|
a.setRetentionControl({category:'campaign',id,resolve:true,protected:id==='held'});
|
|
if (id !== 'new-resolution') a.db.prepare("UPDATE retention_controls SET resolved_at=? WHERE record_id=?").run(old,id);
|
|
}
|
|
a.db.prepare("UPDATE deployments SET started_at=? WHERE id='changed'").run(now);
|
|
// Failure retention works independently of normal campaign retention.
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:0,failureKeepDays:7});
|
|
expect(a.retentionPreview().find(p=>p.category==='campaign')?.records).toEqual([{id:'resolved'}]);
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:7});
|
|
const insert = a.db.prepare("INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)");
|
|
a.db.transaction(() => { for (let n=0;n<600;n++) insert.run(`old-${n}`,old); })();
|
|
expect(a.pruneHistory('campaign')).toBe(500);
|
|
expect(a.pruneHistory('campaign')).toBe(101);
|
|
expect(a.db.prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([
|
|
{id:'changed'},{id:'held'},{id:'new-resolution'},{id:'unresolved'}
|
|
]);
|
|
});
|
|
it('expires file bodies without silently turning rollback into file deletion', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('p','promote',?,?)").run(old,old);
|
|
a.db.prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before,content_after) VALUES('p','s','updated','test.cfg',?,?)").run(Buffer.from('before'),Buffer.from('after'));
|
|
a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 0,fileContentKeepDays: 1 });
|
|
expect(a.retentionPreview().find(p=>p.category==='file-content')?.payloadBytes).toBe(11);
|
|
expect(a.pruneHistory('file-content')).toBe(1);
|
|
expect(a.db.prepare("SELECT content_before,content_after FROM deployment_files").get()).toEqual({ content_before:null,content_after:null });
|
|
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
|
|
await expect(rollbackPromoteDeployment('p')).rejects.toThrow('expired');
|
|
expect(a.pruneHistory('file-content')).toBe(0);
|
|
});
|
|
it('requires explicit failure resolution and preserves recovery slots even after resolution', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO servers(name) VALUES('s')").run();
|
|
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('failure','s','restart','failed','failed',?,?,'test')").run(old,old);
|
|
a.db.prepare("INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES('s','failure',?)").run(old);
|
|
a.saveFleetSettings({ ...a.fleetSettings(),provisioningKeepDays: 1,failureKeepDays: 1 });
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.setRetentionControl({ category:'provisioning',id:'failure',resolve:true });
|
|
a.db.prepare("UPDATE retention_controls SET resolved_at=?").run(old);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.db.prepare("UPDATE restart_rollout SET healthy_at=?").run(new Date().toISOString());
|
|
expect(a.pruneHistory('provisioning')).toBe(1);
|
|
});
|
|
it('compacts action history but preserves idempotency and terminal receipts', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name:'Host' });
|
|
const op = hosts.queueHostOperation(host.id,'files.stage',{ file:'large' },{ idempotencyKey:'original' });
|
|
hosts.reportHostOperation(host.id,op.id,{ state:'completed',result:{ done:true } });
|
|
a.db.prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?').run(old,op.id);
|
|
a.saveFleetSettings({ ...a.fleetSettings(),actionKeepDays:1 });
|
|
expect(a.pruneHistory('operation')).toBe(1);
|
|
expect(hosts.listHostOperations(host.id)).toHaveLength(0);
|
|
expect(hosts.queueHostOperation(host.id,'files.stage',{ file:'large' },{ idempotencyKey:'original' }).id).toBe(op.id);
|
|
expect(()=>hosts.queueHostOperation(host.id,'files.stage',{ file:'different' },{ idempotencyKey:'original' })).toThrow('different request');
|
|
expect(hosts.pendingHostOperations(host.id)).toHaveLength(0);
|
|
expect(hosts.reportHostOperation(host.id,op.id,{state:'running'}).state).toBe('completed');
|
|
});
|
|
it('deletes old maintenance jobs while preserving update suppression evidence and protected jobs', async () => {
|
|
const a = await api();
|
|
for (const kind of ['legacy','schedule','update']) a.db.prepare("INSERT INTO maintenance_jobs(server,state,kind,requested_at,finished_at) VALUES(?,'done',?,?,?)").run(kind,kind,old,old);
|
|
a.setRetentionControl({category:'maintenance',id:'1',protected:true});
|
|
expect(a.pruneHistory('maintenance')).toBe(1);
|
|
expect(a.db.prepare('SELECT kind FROM maintenance_jobs ORDER BY id').all()).toEqual([{kind:'legacy'},{kind:'update'}]);
|
|
});
|
|
it('keeps current and referenced artifact versions and honors the minimum version count', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO scripts(id,name,target_rel_path,current_version_id) VALUES(1,'test','test.gsc',1)").run();
|
|
for(let n=1;n<=4;n++) a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content,uploaded_at) VALUES(?,1,?,'sha',1,X'01',?)").run(n,n,old);
|
|
a.saveFleetSettings({...a.fleetSettings(),versionKeepCount:2});
|
|
expect(a.pruneHistory('version')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM script_versions ORDER BY id').all()).toEqual([{id:1},{id:3},{id:4}]);
|
|
});
|
|
it('prunes metrics and diagnostics while leaving unlimited histories intact', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,'s','t5','kino',0,4)").run(Date.parse(old));
|
|
a.saveFleetSettings({...a.fleetSettings(),metricsKeepDays:0});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({n:1});
|
|
a.saveFleetSettings({...a.fleetSettings(),metricsKeepDays:1});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({n:0});
|
|
});
|
|
|
|
it('prunes closed matches and audit events without deleting active matches or protected audit records', async () => {
|
|
const a = await api();
|
|
for (const status of ['active','completed','interrupted']) a.db.prepare(`INSERT INTO leaderboard_matches(server,game,map,map_key,map_name,started_at,last_received_at,status,last_event,last_fingerprint)
|
|
VALUES('s','t5','kino','kino','Kino',?,?,?,'game_end','test')`).run(old,old,status);
|
|
// Preview initializes the optional authentication schema.
|
|
a.retentionPreview();
|
|
a.db.prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)").run(Date.parse(old));
|
|
a.db.prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)").run(Date.parse(old));
|
|
a.setRetentionControl({category:'audit',id:'1',protected:true});
|
|
a.saveFleetSettings({...a.fleetSettings(),matchKeepDays:1,auditKeepDays:1});
|
|
expect(a.pruneHistory('match')).toBe(2);
|
|
expect(a.pruneHistory('audit')).toBe(1);
|
|
expect(a.db.prepare('SELECT status FROM leaderboard_matches').all()).toEqual([{status:'active'}]);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
it('keeps active beta evidence while expiring closed snapshots and then closed history', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'s','s.gsc')").run();
|
|
a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content) VALUES(1,1,1,'sha',3,X'010203')").run();
|
|
for (const [id,state] of [[1,'running'],[2,'rolled-back']] as const) {
|
|
a.db.prepare("INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at,closed_at) VALUES(?,1,1,'test',?,?,?)").run(id,state,old,old);
|
|
a.db.prepare("INSERT INTO beta_revisions(beta_id,revision,started_at) VALUES(?,1,?)").run(id,old);
|
|
a.db.prepare("INSERT INTO beta_revision_files(beta_id,revision,script_id,name,kind,path,version_id,version,sha256,content) VALUES(?,1,1,'s','script','s.gsc',1,1,'sha',X'010203')").run(id);
|
|
}
|
|
a.saveFleetSettings({...a.fleetSettings(),fileContentKeepDays:1});
|
|
expect(a.pruneHistory('beta-file-content')).toBe(1);
|
|
expect(a.db.prepare('SELECT beta_id,length(content) AS bytes FROM beta_revision_files ORDER BY beta_id').all()).toEqual([{beta_id:1,bytes:3},{beta_id:2,bytes:0}]);
|
|
a.saveFleetSettings({...a.fleetSettings(),betaKeepDays:1});
|
|
expect(a.pruneHistory('beta')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM betas').all()).toEqual([{id:1}]);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
it('applies minimum count and holds to routine cleanup and its preview consistently', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({name:'Host'});
|
|
for (let n=1;n<=3;n++) a.db.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,observation_success,routine) VALUES(?,?,'files.read','{}','completed',?,?,1,1)").run(String(n),host.id,old,old);
|
|
a.setRetentionControl({category:'observation',id:'1',protected:true});
|
|
a.saveFleetSettings({...a.fleetSettings(),historyKeepCount:1});
|
|
expect(a.retentionPreview().find(p=>p.category==='observation')?.eligible).toBe(1);
|
|
expect(a.pruneHistory('observation')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM ec_host_operations ORDER BY id').all()).toEqual([{id:'1'},{id:'3'}]);
|
|
});
|
|
it('never removes the calibration window when choosing a shorter resource retention policy', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({name:'Host'});
|
|
a.db.prepare('INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,?)').run(host.id,old);
|
|
a.db.prepare("INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,'i',1,'startup',?,1,1024)").run(host.id,old);
|
|
a.saveFleetSettings({...a.fleetSettings(),hostResourceKeepDays:0,instanceResourceKeepDays:0});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({n:1});
|
|
a.saveFleetSettings({...a.fleetSettings(),hostResourceKeepDays:1,instanceResourceKeepDays:1});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({n:0});
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM ec_host_resource_samples').get()).toEqual({n:0});
|
|
});
|
|
it('previews draft retention without saving it and restricts retention tools to administrators', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('preview','drift',?,?)").run(old,old);
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:0});
|
|
const {POST} = await import('../../src/routes/api/settings/retention/preview/+server.js');
|
|
const response = await POST({request:new Request('http://localhost/api/settings/retention/preview',{method:'POST',body:JSON.stringify({...a.fleetSettings(),deployKeepDays:1})})});
|
|
expect(response.status).toBe(200);
|
|
expect((await response.json()).policies.find((p:{category:string})=>p.category==='campaign').eligible).toBe(1);
|
|
expect(a.fleetSettings().deployKeepDays).toBe(0);
|
|
expect(a.db.prepare('SELECT id FROM deployments').get()).toEqual({id:'preview'});
|
|
const {routePermission} = await import('../../src/lib/server/auth/access.js');
|
|
expect(routePermission('/api/settings/retention','GET')).toBe('admin');
|
|
expect(routePermission('/api/settings/retention/preview','POST')).toBe('admin');
|
|
});
|
|
it('protects cancelled workflows and linked file history through ambiguous and unreconciled receipts', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host=hosts.createHost({name:'Host'});
|
|
a.db.prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)").run(host.id,old);
|
|
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','restart','cancelled','cancelled',?,?,'test')").run(old,old);
|
|
const command=hosts.queueHostOperation(host.id,'instance.restart',{stageId:'campaign'});
|
|
a.db.prepare("UPDATE ec_host_operations SET state='unknown' WHERE id=?").run(command.id);
|
|
a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,files_json) VALUES('cancelled','instance',1,?,'[{\"id\":1}]')").run(command.id);
|
|
a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)").run(old,old);
|
|
a.db.prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')").run();
|
|
a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:1,provisioningKeepDays:1,fileContentKeepDays:1});
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
expect(a.pruneHistory('file-content')).toBe(0);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.db.prepare("UPDATE ec_host_operations SET state='completed' WHERE id=?").run(command.id);
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.db.prepare("UPDATE native_provisioning SET applied_at=? WHERE deployment_id='cancelled'").run(new Date().toISOString());
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
expect(a.pruneHistory('provisioning')).toBe(1);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|