502 lines
21 KiB
TypeScript
502 lines
21 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { parseActivityQuery } from '../../src/lib/activity.js';
|
|
import type { ActivityQuery } from '../../src/lib/activity.js';
|
|
|
|
let fixture: Fixture;
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
});
|
|
afterEach(async () => {
|
|
vi.restoreAllMocks();
|
|
await fixture.cleanup();
|
|
});
|
|
const server = 't5_zombie_theater_1';
|
|
const at = '2026-10-07T12:00:00.000Z';
|
|
|
|
async function setup() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { createHost } = await import('../../src/lib/server/hosts/index.js');
|
|
const { activityHistory } = await import('../../src/lib/server/activity.js');
|
|
const host = createHost({ name: 'Dallas host' });
|
|
const other = createHost({ name: 'Denver host' });
|
|
const handle = db();
|
|
const insert = handle.prepare(`INSERT INTO server_deployments
|
|
(id,server,action,state,stage,started_at,triggered_by,server_deleted_at,observation_json,error)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?)`);
|
|
function operation(
|
|
id: string,
|
|
options: {
|
|
state?: string;
|
|
action?: string;
|
|
server?: string;
|
|
source?: string;
|
|
at?: string;
|
|
deleted?: string;
|
|
} = {}
|
|
) {
|
|
insert.run(
|
|
id,
|
|
options.server ?? server,
|
|
options.action ?? 'restart',
|
|
options.state ?? 'done',
|
|
'Private stage token=secret-stage',
|
|
options.at ?? at,
|
|
options.source ?? 'ui',
|
|
options.deleted ?? null,
|
|
'{"token":"secret-observation"}',
|
|
'secret-error'
|
|
);
|
|
}
|
|
function remote(
|
|
id: string,
|
|
options: {
|
|
kind?: string;
|
|
state?: string;
|
|
key?: string;
|
|
routine?: number;
|
|
success?: number;
|
|
expired?: string;
|
|
host?: string;
|
|
instance?: string;
|
|
} = {}
|
|
) {
|
|
handle
|
|
.prepare(
|
|
`INSERT INTO ec_host_operations (id,host_id,kind,instance_id,idempotency_key,payload,result,state,created_at,updated_at,
|
|
routine,observation_success,server_name,history_expired_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`
|
|
)
|
|
.run(
|
|
id,
|
|
options.host ?? host.id,
|
|
options.kind ?? 'instance.restart',
|
|
options.instance ?? null,
|
|
options.key ?? null,
|
|
'{"token":"secret-payload"}',
|
|
'{"token":"secret-result"}',
|
|
options.state ?? 'completed',
|
|
at,
|
|
at,
|
|
options.routine ?? 0,
|
|
options.success ?? 0,
|
|
server,
|
|
options.expired ?? null
|
|
);
|
|
}
|
|
function link(operationId: string, kind: string, id: string) {
|
|
handle
|
|
.prepare(
|
|
'INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,?,?,?)'
|
|
)
|
|
.run(operationId, kind, id, server);
|
|
}
|
|
function maintenance(id: number, state = 'done', operationId: string | null = null) {
|
|
handle
|
|
.prepare(
|
|
`INSERT INTO maintenance_jobs(id,server,state,target_revision,requested_at,kind,origin,provisioning_id,detail)
|
|
VALUES(?,?,?,123,?,'update','manual',?,'secret-maintenance-detail')`
|
|
)
|
|
.run(id, server, state, at, operationId);
|
|
}
|
|
function incident(id: string, kind = 'health', state = 'resolved') {
|
|
handle
|
|
.prepare(
|
|
`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
|
|
VALUES(?,?,?,?,'secret-summary','{"credential":"secret-details"}',?,?)`
|
|
)
|
|
.run(id, server, kind, state, at, at);
|
|
}
|
|
function instance(id: string, hostId = host.id) {
|
|
handle
|
|
.prepare(
|
|
`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
|
VALUES(?,?,?,'library','image',28960,?)`
|
|
)
|
|
.run(id, server, hostId, at);
|
|
}
|
|
return {
|
|
handle,
|
|
activityHistory,
|
|
operation,
|
|
remote,
|
|
link,
|
|
maintenance,
|
|
incident,
|
|
instance,
|
|
host,
|
|
other
|
|
};
|
|
}
|
|
|
|
describe('Activity query parameters', () => {
|
|
it('keeps exact identities and valid filters while bounding pagination and ignoring invalid enum values', () => {
|
|
expect(
|
|
parseActivityQuery(
|
|
new URLSearchParams(
|
|
'server=t5_zombie_theater_1&host=a&type=restart&source=maintenance&state=failed&days=7&q=Kino&limit=999&offset=20'
|
|
)
|
|
)
|
|
).toMatchObject({
|
|
server,
|
|
host: 'a',
|
|
type: 'restart',
|
|
source: 'maintenance',
|
|
state: 'failed',
|
|
days: 7,
|
|
q: 'Kino',
|
|
limit: 100,
|
|
offset: 20
|
|
});
|
|
expect(
|
|
parseActivityQuery(
|
|
new URLSearchParams(
|
|
'type=sql&state=bad&source=bad&days=NaN&limit=-5&offset=Infinity'
|
|
)
|
|
)
|
|
).toEqual({ type: undefined, state: undefined, source: undefined, limit: 25, offset: 0 });
|
|
});
|
|
});
|
|
|
|
describe('unified retained Activity', () => {
|
|
it('excludes everything in Maintenance history, including shared recovery campaigns, without changing that history', async () => {
|
|
const api = await setup();
|
|
api.maintenance(1);
|
|
api.maintenance(2);
|
|
api.handle.prepare("UPDATE maintenance_jobs SET kind='schedule' WHERE id=2").run();
|
|
api.operation('shared');
|
|
api.maintenance(3, 'failed', 'shared');
|
|
api.link('shared', 'campaign', 'release');
|
|
api.incident('health');
|
|
api.incident('drift', 'drift');
|
|
api.remote('check', { kind: 'instance.inspect', routine: 1 });
|
|
for (const [id, trigger] of [
|
|
['recovery', 'watchdog'],
|
|
['release', 'ui']
|
|
]) {
|
|
api.handle
|
|
.prepare(
|
|
"INSERT INTO deployments(id,action,started_at,triggered_by) VALUES(?,'drift',?,?)"
|
|
)
|
|
.run(id, at, trigger);
|
|
api.handle
|
|
.prepare(
|
|
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES(?,?,1,'done')"
|
|
)
|
|
.run(id, server);
|
|
}
|
|
const maintenance = api.activityHistory({ source: 'maintenance' });
|
|
expect(maintenance.total).toBe(6);
|
|
const query = { excludeProvisioning: true, excludeMaintenance: true, limit: 1 };
|
|
const first = api.activityHistory(query);
|
|
const second = api.activityHistory({ ...query, offset: 1 });
|
|
expect(first.total).toBe(2);
|
|
expect(second.total).toBe(2);
|
|
expect([...first.rows, ...second.rows].map((row) => row.id).sort()).toEqual([
|
|
`campaign:release:${server}`,
|
|
'host:check'
|
|
]);
|
|
for (const source of ['maintenance', 'recovery'] as const) {
|
|
expect(api.activityHistory({ ...query, source }).total).toBe(0);
|
|
}
|
|
expect(api.activityHistory({ ...query, source: 'campaign' }).total).toBe(1);
|
|
expect(api.activityHistory({ source: 'maintenance' })).toEqual(maintenance);
|
|
expect(api.activityHistory({ server }).total).toBe(8);
|
|
});
|
|
|
|
it('excludes provisioning jobs from log totals and pages regardless of action or trigger, while retaining contextual history', async () => {
|
|
const api = await setup();
|
|
for (const action of ['create', 'start', 'restart', 'rebuild', 'stop', 'destroy']) {
|
|
for (const source of [
|
|
'ui',
|
|
'autoscaling:policy',
|
|
'maintenance:1',
|
|
'campaign:release',
|
|
'beta:1',
|
|
'watchdog:finding'
|
|
]) {
|
|
api.operation(`${action}-${source}`, { action, source });
|
|
}
|
|
}
|
|
api.remote('check', { kind: 'instance.inspect', routine: 1 });
|
|
api.remote('standalone-restart', { kind: 'instance.restart' });
|
|
api.maintenance(3);
|
|
api.handle.prepare("UPDATE maintenance_jobs SET kind='schedule' WHERE id=3").run();
|
|
api.incident('finding');
|
|
api.maintenance(2);
|
|
const query = { excludeProvisioning: true, limit: 2 };
|
|
const first = api.activityHistory(query);
|
|
const last = api.activityHistory({ ...query, offset: 2 });
|
|
expect(first.total).toBe(3);
|
|
expect(last.total).toBe(3);
|
|
expect(first.rows).toHaveLength(2);
|
|
expect(last.rows).toHaveLength(1);
|
|
expect([...first.rows, ...last.rows].map((row) => row.id).sort()).toEqual([
|
|
'host:check',
|
|
'maintenance:2',
|
|
'recovery:finding'
|
|
]);
|
|
expect(api.activityHistory({ ...query, type: 'create' }).total).toBe(0);
|
|
expect(api.activityHistory({ ...query, type: 'restart' }).total).toBe(0);
|
|
expect(api.activityHistory({ ...query, source: 'autoscaling' }).total).toBe(0);
|
|
expect(api.activityHistory({ server }).total).toBe(41);
|
|
});
|
|
|
|
it('paginates the full union with exact totals, stable ties and removed-server history', async () => {
|
|
const api = await setup();
|
|
api.handle.transaction(() => {
|
|
for (let i = 0; i < 130; i++) api.operation(`life-${String(i).padStart(3, '0')}`);
|
|
for (let i = 0; i < 33; i++) api.remote(`remote-${i}`);
|
|
api.operation('removed', { deleted: at, at: '2025-01-01T00:00:00.000Z' });
|
|
api.incident('finding');
|
|
api.maintenance(1);
|
|
})();
|
|
const all = [];
|
|
for (let offset = 0; offset < 166; offset += 25) {
|
|
const page = api.activityHistory({ offset });
|
|
expect(page.total).toBe(166);
|
|
expect(page.rows.length).toBeLessThanOrEqual(25);
|
|
all.push(...page.rows);
|
|
}
|
|
expect(new Set(all.map((row) => row.id)).size).toBe(166);
|
|
expect(all.at(-1)).toMatchObject({ id: 'provisioning:removed', serverDeletedAt: at });
|
|
expect(api.activityHistory({ offset: 99_999 }).offset).toBe(150);
|
|
});
|
|
|
|
it('filters before pagination by exact server, host, state, type, days and friendly search', async () => {
|
|
const api = await setup();
|
|
for (let i = 0; i < 40; i++) api.operation(`other-${i}`, { server: 't5_zombie_moon_1' });
|
|
api.operation('match', { state: 'failed' });
|
|
api.operation('old', { state: 'failed', at: '2025-01-01T00:00:00.000Z' });
|
|
api.instance('instance');
|
|
api.handle
|
|
.prepare(
|
|
'INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)'
|
|
)
|
|
.run('match', 'instance');
|
|
const filters: ActivityQuery = {
|
|
server,
|
|
host: api.host.id,
|
|
state: 'failed',
|
|
type: 'restart',
|
|
q: 'Kino',
|
|
days: 7,
|
|
limit: 1
|
|
};
|
|
const page = api.activityHistory(filters, Date.parse(at));
|
|
expect(page.total).toBe(1);
|
|
expect(page.rows[0]).toMatchObject({
|
|
id: 'provisioning:match',
|
|
hostId: api.host.id,
|
|
hostName: 'Dallas host'
|
|
});
|
|
expect(api.activityHistory({ ...filters, server: 'Kino' }, Date.parse(at)).total).toBe(0);
|
|
expect(api.activityHistory({ ...filters, host: api.other.id }, Date.parse(at)).total).toBe(
|
|
0
|
|
);
|
|
expect(api.activityHistory({ q: "' OR 1=1 --" }).total).toBe(0);
|
|
expect(api.activityHistory({ q: 'Dallas' }).total).toBe(1);
|
|
});
|
|
|
|
it('compares instants across SQLite and ISO timestamp formats for date ranges and ordering', async () => {
|
|
const api = await setup();
|
|
api.operation('iso-noon', { at: '2026-10-07T12:00:00.000Z' });
|
|
api.operation('sql-afternoon', { at: '2026-10-07 13:00:00' });
|
|
api.operation('sql-yesterday', { at: '2026-10-06 14:00:00' });
|
|
api.operation('too-old', { at: '2026-10-06T11:00:00.000Z' });
|
|
const page = api.activityHistory({ days: 1 }, Date.parse(at));
|
|
expect(page.rows.map((row) => row.id)).toEqual([
|
|
'provisioning:sql-afternoon',
|
|
'provisioning:iso-noon',
|
|
'provisioning:sql-yesterday'
|
|
]);
|
|
expect(page.total).toBe(3);
|
|
});
|
|
|
|
it('merges maintenance verification into its lifecycle row and retains every trigger membership', async () => {
|
|
const api = await setup();
|
|
api.operation('shared');
|
|
api.maintenance(1, 'failed', 'shared');
|
|
api.link('shared', 'maintenance', '1');
|
|
api.link('shared', 'campaign', 'expired-campaign');
|
|
api.link('shared', 'beta', '99');
|
|
const row = api.activityHistory().rows[0];
|
|
expect(api.activityHistory().total).toBe(1);
|
|
expect(row).toMatchObject({
|
|
id: 'provisioning:shared',
|
|
type: 'update',
|
|
state: 'failed',
|
|
source: 'maintenance',
|
|
sources: ['maintenance', 'campaign', 'beta'],
|
|
href: '/provisioning/shared',
|
|
detail: 'The restart completed, but maintenance verification failed.'
|
|
});
|
|
for (const source of ['maintenance', 'campaign', 'beta'] as const)
|
|
expect(api.activityHistory({ source }).total).toBe(1);
|
|
api.handle.prepare("UPDATE maintenance_jobs SET state='restarting' WHERE id=1").run();
|
|
expect(api.activityHistory().rows[0]).toMatchObject({
|
|
state: 'running',
|
|
rawState: 'verifying'
|
|
});
|
|
api.handle.prepare("UPDATE maintenance_jobs SET state='done' WHERE id=1").run();
|
|
expect(api.activityHistory().rows[0].state).toBe('succeeded');
|
|
});
|
|
|
|
it('includes recovery in the maintenance umbrella and keeps findings distinct from recovery actions', async () => {
|
|
const api = await setup();
|
|
api.operation('recovery', { source: 'watchdog:finding', action: 'rebuild' });
|
|
api.incident('finding', 'drift');
|
|
api.maintenance(1);
|
|
expect(api.activityHistory({ source: 'maintenance' }).total).toBe(3);
|
|
const recovery = api.activityHistory({ source: 'recovery' });
|
|
expect(recovery.total).toBe(2);
|
|
expect(recovery.rows.map((row) => row.type).sort()).toEqual(['drift', 'rebuild']);
|
|
expect(api.activityHistory({ source: 'provisioning' }).total).toBe(0);
|
|
});
|
|
|
|
it('merges legacy source links without applying a retried verification result to the earlier operation', async () => {
|
|
const api = await setup();
|
|
api.operation('old', { source: 'maintenance:1', at: '2026-10-06T12:00:00.000Z' });
|
|
api.operation('new', { source: 'maintenance:1' });
|
|
api.maintenance(1, 'failed');
|
|
const rows = api.activityHistory().rows;
|
|
expect(rows).toHaveLength(2);
|
|
expect(rows.find((row) => row.id === 'provisioning:old')?.state).toBe('succeeded');
|
|
expect(rows.find((row) => row.id === 'provisioning:new')?.state).toBe('failed');
|
|
});
|
|
|
|
it('deduplicates current command pointers and historical idempotency attempts without dropping independent work', async () => {
|
|
const api = await setup();
|
|
api.operation('operation');
|
|
api.instance('instance');
|
|
api.remote('pointed');
|
|
api.remote('staged');
|
|
api.remote('migration');
|
|
api.handle
|
|
.prepare(
|
|
`INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,stage_operation_id)
|
|
VALUES('operation','instance',1,'pointed','staged')`
|
|
)
|
|
.run();
|
|
api.handle
|
|
.prepare(
|
|
`INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,remove_operation_id)
|
|
VALUES('operation',?,'instance','migration')`
|
|
)
|
|
.run(api.host.id);
|
|
for (const key of [
|
|
'operation',
|
|
'operation:stage',
|
|
'operation:migration-create:1',
|
|
'operation:migrate-remove:2'
|
|
])
|
|
api.remote(`linked-${key}`, { key });
|
|
api.remote('independent');
|
|
api.remote('not-a-retry', { key: 'operation:migration-create:not-a-number' });
|
|
api.remote('source-retained-away', { key: 'expired-operation' });
|
|
expect(
|
|
api
|
|
.activityHistory()
|
|
.rows.map((row) => row.id)
|
|
.sort()
|
|
).toEqual([
|
|
'host:independent',
|
|
'host:not-a-retry',
|
|
'host:source-retained-away',
|
|
'provisioning:operation'
|
|
]);
|
|
});
|
|
|
|
it('keeps failed host observations, hides successful routine polls and expired records, and never projects private data', async () => {
|
|
const api = await setup();
|
|
api.operation('operation');
|
|
api.incident('finding');
|
|
api.maintenance(1);
|
|
api.remote('routine', { kind: 'instance.inspect', routine: 1, success: 1 });
|
|
api.remote('bad-check', { kind: 'instance.inspect', routine: 1 });
|
|
api.remote('failed-check', { kind: 'instance.inspect', routine: 1, state: 'failed' });
|
|
api.remote('expired', { expired: at });
|
|
const result = api.activityHistory();
|
|
expect(
|
|
result.rows
|
|
.filter((row) => row.source === 'host')
|
|
.map((row) => row.id)
|
|
.sort()
|
|
).toEqual(['host:bad-check', 'host:failed-check']);
|
|
expect(result.rows.find((row) => row.id === 'host:bad-check')).toMatchObject({
|
|
state: 'failed',
|
|
rawState: 'observation-failed',
|
|
href: '/hosts/operations?operation=bad-check&filter=all'
|
|
});
|
|
expect(JSON.stringify(result)).not.toContain('secret-');
|
|
expect(JSON.stringify(result)).not.toMatch(/payload|observation_json|result|credential/);
|
|
});
|
|
|
|
it('retains file campaigns and beta tests as distinct work with friendly titles', async () => {
|
|
const api = await setup();
|
|
api.handle
|
|
.exec(`INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'Script','script.gsc');
|
|
INSERT INTO script_versions(id,script_id,version,sha256,size) VALUES(1,1,1,'hash',0);
|
|
INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at) VALUES(1,1,1,'ui','promoted','${at}');
|
|
INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(1,'${server}',0,'ready');
|
|
INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote','${at}');
|
|
INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state,server_deleted_at) VALUES('campaign','${server}',1,'done','${at}');`);
|
|
const rows = api.activityHistory().rows;
|
|
expect(rows).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
source: 'campaign',
|
|
title: 'Promote files',
|
|
detail: '1 file changed.',
|
|
state: 'succeeded',
|
|
serverDeletedAt: at,
|
|
href: '/campaigns/campaign'
|
|
}),
|
|
expect.objectContaining({
|
|
source: 'beta',
|
|
title: 'Beta test #1',
|
|
state: 'succeeded',
|
|
href: '/betas/1'
|
|
})
|
|
])
|
|
);
|
|
api.handle.prepare("UPDATE betas SET state='running' WHERE id=1").run();
|
|
for (const stage of ['preparing', 'staging', 'testing', 'observing']) {
|
|
api.handle.prepare('UPDATE beta_servers SET stage=? WHERE beta_id=1').run(stage);
|
|
expect(api.activityHistory({ source: 'beta' }).rows[0].state).toBe('running');
|
|
}
|
|
api.handle
|
|
.prepare("UPDATE deployments SET triggered_by='watchdog' WHERE id='campaign'")
|
|
.run();
|
|
expect(api.activityHistory({ source: 'maintenance' }).rows[0]).toMatchObject({
|
|
sources: ['recovery', 'campaign']
|
|
});
|
|
expect(api.activityHistory({ source: 'recovery' }).total).toBe(1);
|
|
expect(api.activityHistory({ source: 'campaign' }).total).toBe(1);
|
|
});
|
|
|
|
it('keeps the shared history query contract and no-store API responses', async () => {
|
|
const api = await setup();
|
|
api.operation('a');
|
|
api.operation('b', { state: 'failed' });
|
|
api.remote('failed-check', { kind: 'instance.inspect', routine: 1, state: 'failed' });
|
|
api.maintenance(1, 'failed');
|
|
api.incident('finding', 'health', 'attention');
|
|
const url = new URL(
|
|
'http://localhost/api/activity?state=failed&limit=1&excludeProvisioning=true&excludeMaintenance=true'
|
|
);
|
|
const { GET } = await import('../../src/routes/api/activity/+server.js');
|
|
const response = GET({ url });
|
|
expect(response.headers.get('cache-control')).toBe('no-store');
|
|
const body = await response.json();
|
|
expect(body).toEqual(
|
|
JSON.parse(JSON.stringify(api.activityHistory(parseActivityQuery(url.searchParams))))
|
|
);
|
|
expect(body.rows[0].id).toBe('host:failed-check');
|
|
const history = await GET({ url: new URL('http://localhost/api/activity') }).json();
|
|
expect(history.rows.map((row: { id: string }) => row.id)).toEqual(
|
|
expect.arrayContaining(['provisioning:a', 'provisioning:b', 'host:failed-check'])
|
|
);
|
|
});
|
|
});
|