706 lines
30 KiB
TypeScript
706 lines
30 KiB
TypeScript
import { putFileBlob } from '../../src/lib/server/db/file-blobs.js';
|
|
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
let fixture: Fixture;
|
|
const old = new Date(Date.now() - 90 * 86400000).toISOString();
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
});
|
|
async function api() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
return {
|
|
db: db(),
|
|
...(await import('../../src/lib/server/retention.js')),
|
|
...(await import('../../src/lib/server/settings.js'))
|
|
};
|
|
}
|
|
it('previews eligible campaigns, respects latest-N and explicit holds, and treats zero as indefinite', async () => {
|
|
const a = await api();
|
|
for (const id of ['a', 'b', 'c'])
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)"
|
|
)
|
|
.run(id, old, old);
|
|
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 1, historyKeepCount: 1 });
|
|
a.setRetentionControl({ category: 'campaign', id: 'a', protected: true });
|
|
expect(a.retentionPreview().find((p) => p.category === 'campaign')?.eligible).toBe(1);
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([
|
|
{ id: 'a' },
|
|
{ id: 'c' }
|
|
]);
|
|
a.setRetentionControl({ category: 'campaign', id: 'a', protected: false });
|
|
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 0, historyKeepCount: 0 });
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
});
|
|
it('seeks expired campaigns and uses indexed dependency checks even with failure retention enabled', async () => {
|
|
const a = await api();
|
|
const now = new Date().toISOString();
|
|
const insert = a.db.prepare(
|
|
"INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)"
|
|
);
|
|
a.db.transaction(() => {
|
|
for (let n = 0; n < 8000; n++) insert.run(`recent-${n}`, now, now);
|
|
})();
|
|
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 7, failureKeepDays: 7 });
|
|
const prepare = vi.spyOn(a.db, 'prepare');
|
|
let sql: string;
|
|
try {
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
sql = prepare.mock.calls.find(([sql]) => sql.startsWith('SELECT o.rowid AS rid'))![0];
|
|
} finally {
|
|
prepare.mockRestore();
|
|
}
|
|
const plan = a.db
|
|
.prepare(`EXPLAIN QUERY PLAN ${sql}`)
|
|
.all({ category: 'campaign', cutoff: old, failureCutoff: old }) as { detail: string }[];
|
|
const details = plan.map((row) => row.detail).join('\n');
|
|
expect(details).toMatch(
|
|
/SEARCH o USING (?:COVERING )?INDEX idx_deployments_started \(started_at<\?\)/
|
|
);
|
|
for (const index of [
|
|
'idx_deployments_rolled_back_by',
|
|
'idx_watchdog_actions_campaign',
|
|
'idx_provisioning_sources_source'
|
|
]) {
|
|
expect(details).toContain(index);
|
|
}
|
|
expect(details).not.toMatch(/SCAN [odas]\b/);
|
|
});
|
|
it('selects resolved failures separately while preserving age, protection and batch limits', async () => {
|
|
const a = await api();
|
|
const now = new Date().toISOString();
|
|
for (const id of ['normal', 'unresolved', 'resolved', 'held', 'new-resolution', 'changed']) {
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)"
|
|
)
|
|
.run(id, old, old);
|
|
if (id !== 'normal')
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES(?,'s',1,'failed')"
|
|
)
|
|
.run(id);
|
|
}
|
|
for (const id of ['resolved', 'held', 'new-resolution', 'changed']) {
|
|
a.setRetentionControl({
|
|
category: 'campaign',
|
|
id,
|
|
resolve: true,
|
|
protected: id === 'held'
|
|
});
|
|
if (id !== 'new-resolution')
|
|
a.db
|
|
.prepare('UPDATE retention_controls SET resolved_at=? WHERE record_id=?')
|
|
.run(old, id);
|
|
}
|
|
a.db.prepare("UPDATE deployments SET started_at=? WHERE id='changed'").run(now);
|
|
// Failure retention works independently of normal campaign retention.
|
|
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 0, failureKeepDays: 7 });
|
|
expect(a.retentionPreview().find((p) => p.category === 'campaign')?.records).toEqual([
|
|
{ id: 'resolved' }
|
|
]);
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 7 });
|
|
const insert = a.db.prepare(
|
|
"INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)"
|
|
);
|
|
a.db.transaction(() => {
|
|
for (let n = 0; n < 600; n++) insert.run(`old-${n}`, old);
|
|
})();
|
|
expect(a.pruneHistory('campaign')).toBe(500);
|
|
expect(a.pruneHistory('campaign')).toBe(101);
|
|
expect(a.db.prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([
|
|
{ id: 'changed' },
|
|
{ id: 'held' },
|
|
{ id: 'new-resolution' },
|
|
{ id: 'unresolved' }
|
|
]);
|
|
});
|
|
it('expires file bodies without silently turning rollback into file deletion', async () => {
|
|
const a = await api();
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO deployments(id,action,started_at,finished_at) VALUES('p','promote',?,?)"
|
|
)
|
|
.run(old, old);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before_hash,content_after_hash) VALUES('p','s','updated','test.cfg',?,?)"
|
|
)
|
|
.run(putFileBlob(Buffer.from('before'), a.db), putFileBlob(Buffer.from('after'), a.db));
|
|
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 0, fileContentKeepDays: 1 });
|
|
expect(a.retentionPreview().find((p) => p.category === 'file-content')?.payloadBytes).toBe(11);
|
|
expect(a.pruneHistory('file-content')).toBe(1);
|
|
expect(
|
|
a.db
|
|
.prepare(
|
|
'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,(SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files'
|
|
)
|
|
.get()
|
|
).toEqual({ content_before: null, content_after: null });
|
|
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
|
|
await expect(rollbackPromoteDeployment('p')).rejects.toThrow('expired');
|
|
expect(a.pruneHistory('file-content')).toBe(0);
|
|
});
|
|
it('requires explicit failure resolution and preserves recovery slots even after resolution', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO servers(name) VALUES('s')").run();
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('failure','s','restart','failed','failed',?,?,'test')"
|
|
)
|
|
.run(old, old);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES('s','failure',?)"
|
|
)
|
|
.run(old);
|
|
a.saveFleetSettings({ ...a.fleetSettings(), provisioningKeepDays: 1, failureKeepDays: 1 });
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.setRetentionControl({ category: 'provisioning', id: 'failure', resolve: true });
|
|
a.db.prepare('UPDATE retention_controls SET resolved_at=?').run(old);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.db.prepare('UPDATE restart_rollout SET healthy_at=?').run(new Date().toISOString());
|
|
expect(a.pruneHistory('provisioning')).toBe(1);
|
|
});
|
|
it('compacts action history but preserves idempotency and terminal receipts', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Host' });
|
|
const op = hosts.queueHostOperation(
|
|
host.id,
|
|
'files.stage',
|
|
{ file: 'large' },
|
|
{ idempotencyKey: 'original' }
|
|
);
|
|
hosts.reportHostOperation(host.id, op.id, { state: 'completed', result: { done: true } });
|
|
a.db.prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?').run(old, op.id);
|
|
a.saveFleetSettings({ ...a.fleetSettings(), actionKeepDays: 1 });
|
|
expect(a.pruneHistory('operation')).toBe(1);
|
|
expect(hosts.listHostOperations(host.id)).toHaveLength(0);
|
|
expect(
|
|
hosts.queueHostOperation(
|
|
host.id,
|
|
'files.stage',
|
|
{ file: 'large' },
|
|
{ idempotencyKey: 'original' }
|
|
).id
|
|
).toBe(op.id);
|
|
expect(() =>
|
|
hosts.queueHostOperation(
|
|
host.id,
|
|
'files.stage',
|
|
{ file: 'different' },
|
|
{ idempotencyKey: 'original' }
|
|
)
|
|
).toThrow('different request');
|
|
expect(hosts.pendingHostOperations(host.id)).toHaveLength(0);
|
|
expect(hosts.reportHostOperation(host.id, op.id, { state: 'running' }).state).toBe('completed');
|
|
});
|
|
it('deletes old maintenance jobs while preserving update suppression evidence and protected jobs', async () => {
|
|
const a = await api();
|
|
for (const kind of ['legacy', 'schedule', 'update'])
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO maintenance_jobs(server,state,kind,requested_at,finished_at) VALUES(?,'done',?,?,?)"
|
|
)
|
|
.run(kind, kind, old, old);
|
|
a.setRetentionControl({ category: 'maintenance', id: '1', protected: true });
|
|
expect(a.pruneHistory('maintenance')).toBe(1);
|
|
expect(a.db.prepare('SELECT kind FROM maintenance_jobs ORDER BY id').all()).toEqual([
|
|
{ kind: 'legacy' },
|
|
{ kind: 'update' }
|
|
]);
|
|
});
|
|
it('keeps current and referenced artifact versions and honors the minimum version count', async () => {
|
|
const a = await api();
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO scripts(id,name,target_rel_path,current_version_id) VALUES(1,'test','test.gsc',1)"
|
|
)
|
|
.run();
|
|
for (let n = 1; n <= 4; n++)
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash,uploaded_at) VALUES(?,1,?,'sha',1,?,?)"
|
|
)
|
|
.run(n, n, putFileBlob(Buffer.from([1]), a.db), old);
|
|
a.saveFleetSettings({ ...a.fleetSettings(), versionKeepCount: 2 });
|
|
expect(a.pruneHistory('version')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM script_versions ORDER BY id').all()).toEqual([
|
|
{ id: 1 },
|
|
{ id: 3 },
|
|
{ id: 4 }
|
|
]);
|
|
});
|
|
it('prunes metrics and diagnostics while leaving unlimited histories intact', async () => {
|
|
const a = await api();
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,'s','t5','kino',0,4)"
|
|
)
|
|
.run(Date.parse(old));
|
|
a.saveFleetSettings({ ...a.fleetSettings(), metricsKeepDays: 0 });
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({ n: 1 });
|
|
a.saveFleetSettings({ ...a.fleetSettings(), metricsKeepDays: 1 });
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({ n: 0 });
|
|
});
|
|
|
|
it('prunes closed matches and audit events without deleting active matches or protected audit records', async () => {
|
|
const a = await api();
|
|
for (const status of ['active', 'completed', 'interrupted'])
|
|
a.db
|
|
.prepare(
|
|
`INSERT INTO leaderboard_matches(server,game,map,map_key,map_name,started_at,last_received_at,status,last_event,last_fingerprint)
|
|
VALUES('s','t5','kino','kino','Kino',?,?,?,'game_end','test')`
|
|
)
|
|
.run(old, old, status);
|
|
// Preview initializes the optional authentication schema.
|
|
a.retentionPreview();
|
|
a.db
|
|
.prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)")
|
|
.run(Date.parse(old));
|
|
a.db
|
|
.prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)")
|
|
.run(Date.parse(old));
|
|
a.setRetentionControl({ category: 'audit', id: '1', protected: true });
|
|
a.saveFleetSettings({ ...a.fleetSettings(), matchKeepDays: 1, auditKeepDays: 1 });
|
|
expect(a.pruneHistory('match')).toBe(2);
|
|
expect(a.pruneHistory('audit')).toBe(1);
|
|
expect(a.db.prepare('SELECT status FROM leaderboard_matches').all()).toEqual([
|
|
{ status: 'active' }
|
|
]);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
it('keeps active beta evidence while expiring closed snapshots and then closed history', async () => {
|
|
const a = await api();
|
|
a.db.prepare("INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'s','s.gsc')").run();
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash) VALUES(1,1,1,'sha',3,?)"
|
|
)
|
|
.run(putFileBlob(Buffer.from([1, 2, 3]), a.db));
|
|
for (const [id, state] of [
|
|
[1, 'running'],
|
|
[2, 'rolled-back']
|
|
] as const) {
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at,closed_at) VALUES(?,1,1,'test',?,?,?)"
|
|
)
|
|
.run(id, state, old, old);
|
|
a.db
|
|
.prepare('INSERT INTO beta_revisions(beta_id,revision,started_at) VALUES(?,1,?)')
|
|
.run(id, old);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO beta_revision_files(beta_id,revision,script_id,name,kind,path,version_id,version,sha256,content_hash) VALUES(?,1,1,'s','script','s.gsc',1,1,'sha',?)"
|
|
)
|
|
.run(id, putFileBlob(Buffer.from([1, 2, 3]), a.db));
|
|
}
|
|
a.saveFleetSettings({ ...a.fleetSettings(), fileContentKeepDays: 1 });
|
|
expect(a.pruneHistory('beta-file-content')).toBe(1);
|
|
expect(
|
|
a.db
|
|
.prepare(
|
|
'SELECT beta_id,COALESCE((SELECT size FROM file_blobs WHERE sha256=content_hash),0) AS bytes FROM beta_revision_files ORDER BY beta_id'
|
|
)
|
|
.all()
|
|
).toEqual([
|
|
{ beta_id: 1, bytes: 3 },
|
|
{ beta_id: 2, bytes: 0 }
|
|
]);
|
|
a.saveFleetSettings({ ...a.fleetSettings(), betaKeepDays: 1 });
|
|
expect(a.pruneHistory('beta')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM betas').all()).toEqual([{ id: 1 }]);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
it('applies minimum count and holds to routine cleanup and its preview consistently', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Host' });
|
|
for (let n = 1; n <= 3; n++)
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,observation_success,routine) VALUES(?,?,'files.read','{}','completed',?,?,1,1)"
|
|
)
|
|
.run(String(n), host.id, old, old);
|
|
a.setRetentionControl({ category: 'observation', id: '1', protected: true });
|
|
a.saveFleetSettings({ ...a.fleetSettings(), historyKeepCount: 1 });
|
|
expect(a.retentionPreview().find((p) => p.category === 'observation')?.eligible).toBe(1);
|
|
expect(a.pruneHistory('observation')).toBe(1);
|
|
expect(a.db.prepare('SELECT id FROM ec_host_operations ORDER BY id').all()).toEqual([
|
|
{ id: '1' },
|
|
{ id: '3' }
|
|
]);
|
|
});
|
|
it('never removes the calibration window when choosing a shorter resource retention policy', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Host' });
|
|
a.db
|
|
.prepare('INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,?)')
|
|
.run(host.id, old);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,'i',1,'startup',?,1,1024)"
|
|
)
|
|
.run(host.id, old);
|
|
a.saveFleetSettings({
|
|
...a.fleetSettings(),
|
|
hostResourceKeepDays: 0,
|
|
instanceResourceKeepDays: 0
|
|
});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({
|
|
n: 1
|
|
});
|
|
a.saveFleetSettings({
|
|
...a.fleetSettings(),
|
|
hostResourceKeepDays: 1,
|
|
instanceResourceKeepDays: 1
|
|
});
|
|
a.pruneSampleHistory();
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({
|
|
n: 0
|
|
});
|
|
expect(a.db.prepare('SELECT count(*) AS n FROM ec_host_resource_samples').get()).toEqual({
|
|
n: 0
|
|
});
|
|
});
|
|
it('previews draft retention without saving it and distinguishes admin reads from writes', async () => {
|
|
const a = await api();
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO deployments(id,action,started_at,finished_at) VALUES('preview','drift',?,?)"
|
|
)
|
|
.run(old, old);
|
|
a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 0 });
|
|
const { POST } = await import('../../src/routes/api/settings/retention/preview/+server.js');
|
|
const response = await POST({
|
|
request: new Request('http://localhost/api/settings/retention/preview', {
|
|
method: 'POST',
|
|
body: JSON.stringify({ ...a.fleetSettings(), deployKeepDays: 1 })
|
|
})
|
|
});
|
|
expect(response.status).toBe(200);
|
|
expect(
|
|
(await response.json()).policies.find(
|
|
(p: { category: string }) => p.category === 'campaign'
|
|
).eligible
|
|
).toBe(1);
|
|
expect(a.fleetSettings().deployKeepDays).toBe(0);
|
|
expect(a.db.prepare('SELECT id FROM deployments').get()).toEqual({ id: 'preview' });
|
|
const { routePermission } = await import('../../src/lib/server/auth/access.js');
|
|
expect(routePermission('/api/settings/retention', 'GET')).toBe('admin-read');
|
|
expect(routePermission('/api/settings/retention', 'POST')).toBe('admin');
|
|
expect(routePermission('/api/settings/retention/preview', 'POST')).toBe('admin-read');
|
|
});
|
|
it('protects cancelled workflows and linked file history through ambiguous and unreconciled receipts', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Host' });
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)"
|
|
)
|
|
.run(host.id, old);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','restart','cancelled','cancelled',?,?,'test')"
|
|
)
|
|
.run(old, old);
|
|
const command = hosts.queueHostOperation(host.id, 'instance.restart', { stageId: 'campaign' });
|
|
a.db.prepare("UPDATE ec_host_operations SET state='unknown' WHERE id=?").run(command.id);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,files_json) VALUES('cancelled','instance',1,?,'[{\"id\":1}]')"
|
|
)
|
|
.run(command.id);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)"
|
|
)
|
|
.run(old, old);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')"
|
|
)
|
|
.run();
|
|
a.saveFleetSettings({
|
|
...a.fleetSettings(),
|
|
deployKeepDays: 1,
|
|
provisioningKeepDays: 1,
|
|
fileContentKeepDays: 1
|
|
});
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
expect(a.pruneHistory('file-content')).toBe(0);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.db.prepare("UPDATE ec_host_operations SET state='completed' WHERE id=?").run(command.id);
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
a.db
|
|
.prepare("UPDATE native_provisioning SET applied_at=? WHERE deployment_id='cancelled'")
|
|
.run(new Date().toISOString());
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
expect(a.pruneHistory('provisioning')).toBe(1);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
|
|
it('retains a cancelled rebuild and its source until the partial-failure receipt is applied', async () => {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Host' });
|
|
a.db.prepare("INSERT INTO servers(name,container_state) VALUES('s','running')").run();
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)"
|
|
)
|
|
.run(host.id, old);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')"
|
|
)
|
|
.run(old, old);
|
|
const command = hosts.queueHostOperation(
|
|
host.id,
|
|
'instance.rebuild',
|
|
{ nextGeneration: 2 },
|
|
{ instanceId: 'instance', generation: 1 }
|
|
);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('cancelled','instance',1,?)"
|
|
)
|
|
.run(command.id);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)"
|
|
)
|
|
.run(old, old);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')"
|
|
)
|
|
.run();
|
|
hosts.reportHostOperation(host.id, command.id, {
|
|
state: 'failed',
|
|
error: 'Replacement creation failed',
|
|
result: {
|
|
reconciled: true,
|
|
rebuildFailure: {
|
|
phase: 'disrupting',
|
|
generation: 1,
|
|
nextGeneration: 2,
|
|
oldContainerMissing: true,
|
|
replacementContainerMissing: true,
|
|
replacementFilesRemoved: true
|
|
}
|
|
}
|
|
});
|
|
a.saveFleetSettings({
|
|
...a.fleetSettings(),
|
|
deployKeepDays: 1,
|
|
provisioningKeepDays: 1,
|
|
historyKeepCount: 0
|
|
});
|
|
expect(a.pruneHistory('campaign')).toBe(0);
|
|
expect(a.pruneHistory('provisioning')).toBe(0);
|
|
expect(
|
|
a.db
|
|
.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'")
|
|
.get()
|
|
).toEqual({ applied_at: null });
|
|
|
|
const { reconcileNativeProvisioningReceipt } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
reconcileNativeProvisioningReceipt('cancelled');
|
|
expect(
|
|
a.db
|
|
.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'")
|
|
.get()
|
|
).toEqual({ applied_at: expect.any(String) });
|
|
expect(a.db.prepare("SELECT container_state FROM servers WHERE name='s'").get()).toEqual({
|
|
container_state: 'missing'
|
|
});
|
|
expect(a.pruneHistory('campaign')).toBe(1);
|
|
expect(a.pruneHistory('provisioning')).toBe(1);
|
|
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
|
|
async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
|
|
const a = await api();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Recovery host' });
|
|
const { config } = await import('../../src/lib/server/config.js');
|
|
hosts.enrollHost(hosts.createEnrollment(host.id).token, {
|
|
protocolVersion: 1,
|
|
dockerRunning: true,
|
|
directoriesWritable: true,
|
|
storageAvailable: 100_000_000_000,
|
|
preparedRuntimeImages: [config.defaultRuntimeImage],
|
|
libraries: [{ id: 'lib', state: 'ready' }]
|
|
});
|
|
const generation = action === 'create' ? 1 : 2;
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,?,'lib','image',1,?)"
|
|
)
|
|
.run(host.id, generation, old);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('recovering','s',?,'awaiting-health','Recovery',?,'test')"
|
|
)
|
|
.run(action === 'create' ? 'create' : 'rebuild', old);
|
|
const sha256 = putFileBlob(Buffer.from('original launch files'), a.db);
|
|
const launch = hosts.queueHostOperation(
|
|
host.id,
|
|
action === 'rebuild' ? 'instance.rebuild' : 'instance.create',
|
|
{
|
|
name: 's',
|
|
files: [{ path: 'configs/server.cfg', sha256, contentSha256: sha256 }]
|
|
},
|
|
{
|
|
instanceId: 'instance',
|
|
generation: action === 'rebuild' ? 1 : generation,
|
|
idempotencyKey: action === 'migration' ? 'recovering:migration-create:0' : 'recovering'
|
|
}
|
|
);
|
|
hosts.reportHostOperation(host.id, launch.id, { state: 'completed' });
|
|
const restart = hosts.queueHostOperation(
|
|
host.id,
|
|
'instance.restart',
|
|
{ unhealthyRecovery: true },
|
|
{ instanceId: 'instance', generation }
|
|
);
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,recovery_step) VALUES('recovering','instance',?,?,'restart')"
|
|
)
|
|
.run(generation, restart.id);
|
|
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
|
|
a.saveFleetSettings({
|
|
...a.fleetSettings(),
|
|
actionKeepDays: 1,
|
|
payloadKeepDays: 1,
|
|
historyKeepCount: 0
|
|
});
|
|
return { ...a, hosts, host, generation, launch, restart, sha256 };
|
|
}
|
|
|
|
it.each(['create', 'rebuild', 'migration'] as const)(
|
|
'keeps original %s launch files and ownership through recovery and releases them after success',
|
|
async (action) => {
|
|
const a = await recoveryLaunch(action);
|
|
expect(a.pruneHistory('operation-payload')).toBe(0);
|
|
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([
|
|
{
|
|
path: 'configs/server.cfg',
|
|
sha256: a.sha256,
|
|
contentBase64: Buffer.from('original launch files').toString('base64')
|
|
}
|
|
]);
|
|
a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' });
|
|
const rebuild = a.hosts.queueHostOperation(
|
|
a.host.id,
|
|
'instance.rebuild',
|
|
{ unhealthyRecovery: true, nextGeneration: a.generation + 1 },
|
|
{ instanceId: 'instance', generation: a.generation }
|
|
);
|
|
a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' });
|
|
a.db
|
|
.prepare(
|
|
"UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'"
|
|
)
|
|
.run(rebuild.id, old);
|
|
a.db
|
|
.prepare("UPDATE native_instances SET generation=generation+1 WHERE id='instance'")
|
|
.run();
|
|
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
|
|
a.pruneHistory('operation-payload');
|
|
a.pruneHistory('operation');
|
|
expect(
|
|
a.db
|
|
.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?')
|
|
.get(a.launch.id)
|
|
).toEqual({ history_expired_at: null });
|
|
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([
|
|
{
|
|
path: 'configs/server.cfg',
|
|
sha256: a.sha256,
|
|
contentBase64: Buffer.from('original launch files').toString('base64')
|
|
}
|
|
]);
|
|
a.db
|
|
.prepare(
|
|
"UPDATE server_deployments SET state='done',finished_at=? WHERE id='recovering'"
|
|
)
|
|
.run(old);
|
|
a.pruneHistory('operation-payload');
|
|
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([
|
|
{ path: 'configs/server.cfg', sha256: a.sha256 }
|
|
]);
|
|
a.pruneHistory('operation');
|
|
expect(
|
|
a.db
|
|
.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?')
|
|
.get(a.launch.id)
|
|
).toEqual({ history_expired_at: expect.any(String) });
|
|
}
|
|
);
|
|
|
|
it.each(['cancelled', 'failed'])(
|
|
'keeps original ownership for a %s recovered creation until a later lifecycle operation supersedes cleanup',
|
|
async (state) => {
|
|
const a = await recoveryLaunch('create');
|
|
a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' });
|
|
const rebuild = a.hosts.queueHostOperation(
|
|
a.host.id,
|
|
'instance.rebuild',
|
|
{ unhealthyRecovery: true, nextGeneration: 2 },
|
|
{ instanceId: 'instance', generation: 1 }
|
|
);
|
|
a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' });
|
|
a.db
|
|
.prepare(
|
|
"UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'"
|
|
)
|
|
.run(rebuild.id, old);
|
|
a.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run();
|
|
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
|
|
a.db
|
|
.prepare("UPDATE server_deployments SET state=?,finished_at=? WHERE id='recovering'")
|
|
.run(state, old);
|
|
const { canCancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(canCancelCreation('recovering')).toBe(true);
|
|
a.pruneHistory('operation');
|
|
expect(
|
|
a.db
|
|
.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?')
|
|
.get(a.launch.id)
|
|
).toEqual({ history_expired_at: null });
|
|
a.db
|
|
.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('replacement','s','restart','done','Done',?,?,'test')"
|
|
)
|
|
.run(old, old);
|
|
expect(canCancelCreation('recovering')).toBe(false);
|
|
a.pruneHistory('operation');
|
|
expect(
|
|
a.db
|
|
.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?')
|
|
.get(a.launch.id)
|
|
).toEqual({ history_expired_at: expect.any(String) });
|
|
}
|
|
);
|