Add filters and context to host operation history
This commit is contained in:
+24
-5
@@ -60,10 +60,20 @@ or enrolling another host needs no manual propagation. Setup waits for an active
|
||||
healthy host with sufficient storage. Failed operations retry automatically with
|
||||
backoff from one minute to fifteen minutes; their errors remain visible in the
|
||||
host's setup panel and **Hosts → Operation history**. This shared history lists
|
||||
operations across hosts, newest first, with a host column and 25 rows per page.
|
||||
operations across hosts, newest first, with 25 rows per page and host, target,
|
||||
and source details. It defaults to **Actions & errors**; **Routine checks** and
|
||||
**All operations** include successful monitoring reads. Older records may lack
|
||||
source details.
|
||||
Disconnected hosts resume when they
|
||||
reconnect. Draining and disabled hosts do not receive new automatic setup work.
|
||||
|
||||
**Settings → Routine operation history retention (days)** controls cleanup of
|
||||
successful monitoring and file-read records. The default is one day; zero keeps
|
||||
them indefinitely. Actions, failures, negative checks, and unresolved operations
|
||||
are retained. Agent 1.4.9 also cleans up successful routine journal records after
|
||||
the controller confirms their final receipt. Older unconfirmed journal entries
|
||||
remain until a replay supplies that confirmation. Cleanup runs in bounded batches.
|
||||
|
||||
An automatic download that has not been delivered to the agent can be replaced
|
||||
when a newer upload becomes available. A delivered operation finishes safely
|
||||
before the host prepares the latest files. Failed or incomplete uploads never
|
||||
@@ -287,8 +297,11 @@ explicit installer `--reenroll` when local credentials already exist.
|
||||
API outline:
|
||||
|
||||
- `GET /api/hosts/:id`: host details and deletion blockers.
|
||||
- `GET /api/hosts/operations?page=1`: operation history across hosts, newest first,
|
||||
with 25 summaries per page and total/page counts. Payloads and results are omitted.
|
||||
- `GET /api/hosts/operations?page=1&filter=important`: operation history across
|
||||
hosts, newest first, with 25 summaries per page and filtered total/page counts.
|
||||
Filters are `important` (default), `routine`, and `all`. Summaries include the
|
||||
server, safe target path or status command, and initiating source when recorded.
|
||||
Payloads and results are omitted.
|
||||
- `DELETE /api/hosts/:id`: administrator requests cleanup of an unused host;
|
||||
returns `202` while remote removal is pending, `200` for an unenrolled host,
|
||||
or `409` when the agent needs updating or unresolved ownership remains.
|
||||
@@ -551,8 +564,14 @@ of which task delayed a previous deployment.
|
||||
Campaign, beta and maintenance observers run independently of provisioning.
|
||||
New requests and completed agent commands wake provisioning; an arrival during a
|
||||
pass schedules another pass. Routine file drift scans run every five minutes;
|
||||
manual checks bypass that interval. Overlapping identical native observations
|
||||
share one operation until it completes, without caching results for later checks.
|
||||
manual checks bypass that interval. Agent 1.4.9 supports one hash manifest per
|
||||
drift scan, avoiding separate list/stat/read operations for unchanged files.
|
||||
Older agents use the existing file checks. Overlapping identical native
|
||||
observations share one operation until it completes. Routine status monitoring
|
||||
and maintenance additionally share successful health results for up to 30 seconds
|
||||
from the request start, preserving the original observation time. Changed server
|
||||
identity or lifecycle intent invalidates reuse. Manual checks and disruption
|
||||
safety checks continue to request fresh evidence.
|
||||
Agent 1.4.5 reserves two observation workers for instance health and read-only RCON,
|
||||
and one for files, logs, and host inventory. Background reads arrive in batches of
|
||||
up to four. File receipt uploads release the instance lock before contacting the
|
||||
|
||||
+3
-1
@@ -5,6 +5,7 @@ import { interruptDeployments } from '$lib/server/provisioning/history.js';
|
||||
import { authenticationHandle } from '$lib/server/auth/guard.js';
|
||||
import { emit } from '$lib/server/pipeline/bus.js';
|
||||
import { startPerformanceMonitor } from '$lib/server/performance.js';
|
||||
import { withOperationSource } from '$lib/server/hosts/operation-context.js';
|
||||
|
||||
let booted = false;
|
||||
function boot(): void {
|
||||
@@ -22,7 +23,8 @@ function boot(): void {
|
||||
export const handle: Handle = async (input) => {
|
||||
boot();
|
||||
const started = performance.now();
|
||||
const response = await authenticationHandle(input);
|
||||
const response = await withOperationSource(`${input.event.request.method} ${input.event.route.id ?? 'Web request'}`,
|
||||
() => authenticationHandle(input));
|
||||
const elapsed = performance.now() - started;
|
||||
response.headers.append('Server-Timing', `app;dur=${elapsed.toFixed(1)}`);
|
||||
if (process.env.EC_PERF_LOG === 'true' && elapsed > 250 && !input.event.route.id?.startsWith('/api/agent/')) {
|
||||
|
||||
+9
-1
@@ -102,9 +102,17 @@ export interface HostOperation {
|
||||
}
|
||||
export interface HostOperationSummary extends Omit<HostOperation, 'payload' | 'result'> {
|
||||
hostName: string | null;
|
||||
serverName: string | null;
|
||||
targetPath: string | null;
|
||||
source: string | null;
|
||||
routine: boolean;
|
||||
}
|
||||
export type HostOperationFilter = 'important' | 'routine' | 'all';
|
||||
export function isHostOperationFilter(value: unknown): value is HostOperationFilter {
|
||||
return typeof value === 'string' && ['important', 'routine', 'all'].includes(value);
|
||||
}
|
||||
export interface HostOperationHistoryPage {
|
||||
operations: HostOperationSummary[]; page: number; pageSize: 25; total: number; totalPages: number;
|
||||
operations: HostOperationSummary[]; page: number; pageSize: 25; total: number; totalPages: number; filter: HostOperationFilter;
|
||||
}
|
||||
|
||||
export interface HostPreparationItem {
|
||||
|
||||
@@ -4,11 +4,12 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt
|
||||
import { isIP } from 'node:net';
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureHostSchema } from './schema.js';
|
||||
import { successfulObservation } from './observations.js';
|
||||
import { successfulObservation, routineObservationSql, importantOperationSql, operationTargetSql, safeOperationTarget } from './observations.js';
|
||||
import { currentOperationSource } from './operation-context.js';
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { recordInstanceResources, validateInstanceResources } from './calibration.js';
|
||||
import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
|
||||
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationHistoryPage, HostOperationState } from '../../hosts.js';
|
||||
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationFilter, HostOperationHistoryPage, HostOperationState } from '../../hosts.js';
|
||||
|
||||
export const AGENT_PROTOCOL_VERSION = 1;
|
||||
export const HEARTBEAT_MAX_AGE_MS = 90_000;
|
||||
@@ -133,7 +134,8 @@ export function deleteHost(id: string): { deleted: boolean; operationId?: string
|
||||
const operationId = randomUUID();
|
||||
const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
|
||||
const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` };
|
||||
handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp);
|
||||
handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,source) VALUES(?,?,'host.decommission',?,'pending',?,?,?)")
|
||||
.run(operationId, id, JSON.stringify(payload), timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200));
|
||||
handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp);
|
||||
handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
|
||||
ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
|
||||
@@ -359,19 +361,26 @@ export function listHostOperations(hostId: string, limit = 100, summary = false)
|
||||
.all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow);
|
||||
}
|
||||
/** Browse retained history without loading command payloads or credential-bearing results. */
|
||||
export function listHostOperationHistory(page = 1): HostOperationHistoryPage {
|
||||
export function listHostOperationHistory(page = 1, filter: HostOperationFilter = 'important'): HostOperationHistoryPage {
|
||||
const handle = database();
|
||||
const pageSize = 25;
|
||||
const where = filter === 'all' ? '' : `WHERE ${filter === 'routine' ? 'o.routine=1' : importantOperationSql}`;
|
||||
return handle.transaction((): HostOperationHistoryPage => {
|
||||
const { total } = handle.prepare('SELECT COUNT(*) AS total FROM ec_host_operations').get() as { total: number };
|
||||
const { total } = handle.prepare(`SELECT COUNT(*) AS total FROM ec_host_operations o ${where}`).get() as { total: number };
|
||||
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
||||
const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1);
|
||||
const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind,
|
||||
o.instance_id AS instanceId,o.generation,o.state,o.error,o.created_at AS createdAt,o.updated_at AS updatedAt
|
||||
o.instance_id AS instanceId,o.generation,o.state,
|
||||
CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0
|
||||
THEN COALESCE(NULLIF(o.error,''),'Observation reported a problem') ELSE o.error END AS error,
|
||||
o.created_at AS createdAt,o.updated_at AS updatedAt,o.routine,o.server_name AS serverName,o.target_path AS targetPath,o.source
|
||||
FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL
|
||||
${where}
|
||||
ORDER BY o.created_at DESC,o.rowid DESC LIMIT ? OFFSET ?`)
|
||||
.all(pageSize, (currentPage - 1) * pageSize) as HostOperationHistoryPage['operations'];
|
||||
return { operations, page: currentPage, pageSize, total, totalPages };
|
||||
return { operations: operations.map(operation => ({ ...operation, routine: !!operation.routine,
|
||||
targetPath: safeOperationTarget(operation.kind, operation.targetPath) })), filter,
|
||||
page: currentPage, pageSize, total, totalPages };
|
||||
})();
|
||||
}
|
||||
|
||||
@@ -428,8 +437,15 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
|
||||
if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
|
||||
if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
|
||||
const id = options.id ?? randomUUID(); const timestamp = now();
|
||||
database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, routingPayload ?? payload, timestamp, timestamp);
|
||||
const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?')
|
||||
.get(options.instanceId) as { server: string } | undefined)?.server ?? null : null;
|
||||
database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name)
|
||||
VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?)`)
|
||||
.run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null,
|
||||
routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName);
|
||||
if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload);
|
||||
database().prepare(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
|
||||
target_path=${operationTargetSql} WHERE o.id=?`).run(id);
|
||||
database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
|
||||
return getHostOperationStatus(id)!;
|
||||
})();
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
export const observationKinds = ['host.inspect', 'instance.inspect', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read'];
|
||||
const commands = ['status', 'getstatus', 'version'];
|
||||
|
||||
export function safeOperationTarget(kind: string, value: string | null): string | null {
|
||||
if (!value || value.length > 1024) return null;
|
||||
const paths = kind === 'files.manifest' ? value.split(', ') : [value];
|
||||
return paths.every(path => path === '.' || !/[\\\x00-\x1f\x7f]/.test(path)
|
||||
&& path.split('/').every(part => part !== '' && part !== '.' && part !== '..')) ? value : null;
|
||||
}
|
||||
|
||||
export function successfulObservation(kind: string, payload: Record<string, unknown>, result: unknown): boolean {
|
||||
const routine = observationKinds.includes(kind) || kind === 'rcon.execute' && commands.includes(String(payload.command));
|
||||
const value = result && typeof result === 'object' ? result as Record<string, unknown> : {};
|
||||
@@ -8,11 +15,24 @@ export function successfulObservation(kind: string, payload: Record<string, unkn
|
||||
}
|
||||
|
||||
/** Uses the operation alias o; serialized payloads may live outside the routing row. */
|
||||
export const routineObservationSql = `(o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')})
|
||||
OR (o.kind='rcon.execute' AND json_extract(COALESCE(
|
||||
(SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')))`;
|
||||
export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')}) THEN 1
|
||||
WHEN o.kind='rcon.execute' THEN json_extract(COALESCE(
|
||||
(SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')
|
||||
ELSE 0 END)`;
|
||||
|
||||
export const successfulObservationSql = `(o.state='completed' AND COALESCE(o.error,'')='' AND ${routineObservationSql}
|
||||
AND CASE WHEN o.result IS NULL THEN 1 WHEN json_valid(o.result)
|
||||
THEN json_extract(o.result,'$.ok') IS NOT 0 AND json_extract(o.result,'$.healthy') IS NOT 0
|
||||
AND COALESCE(json_extract(o.result,'$.error'),'')='' ELSE 0 END)`;
|
||||
|
||||
export const importantOperationSql = `(o.routine=0 OR o.state IN ('failed','unknown')
|
||||
OR (o.state='completed' AND o.observation_success=0) OR COALESCE(o.error,'')!='')`;
|
||||
|
||||
const payloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
|
||||
/** Deliberately exclude arbitrary RCON commands and all file contents/credentials. */
|
||||
export const operationTargetSql = `CASE
|
||||
WHEN o.kind IN ('files.read','files.stat','files.list','logs.read') THEN
|
||||
CASE WHEN json_type(${payloadSql},'$.path')='text' THEN substr(json_extract(${payloadSql},'$.path'),1,1024) END
|
||||
WHEN o.kind='files.manifest' THEN substr((SELECT group_concat(value,', ') FROM json_each(${payloadSql},'$.roots') WHERE type='text'),1,1024)
|
||||
WHEN o.kind='rcon.execute' AND ${routineObservationSql} THEN json_extract(${payloadSql},'$.command')
|
||||
ELSE NULL END`;
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
|
||||
const operationSource = new AsyncLocalStorage<string>();
|
||||
|
||||
/** Carry the initiating request or background task through asynchronous work. */
|
||||
export function withOperationSource<T>(source: string, work: () => T): T {
|
||||
return operationSource.run(source, work);
|
||||
}
|
||||
|
||||
export function currentOperationSource(): string | null {
|
||||
return operationSource.getStore() ?? null;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
import { successfulObservationSql } from './observations.js';
|
||||
import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
|
||||
|
||||
const initialized = new WeakSet<Database.Database>();
|
||||
export function ensureHostSchema(handle: Database.Database): void {
|
||||
@@ -30,6 +30,7 @@ export function ensureHostSchema(handle: Database.Database): void {
|
||||
CHECK (state IN ('pending','accepted','running','completed','failed','unknown')),
|
||||
result TEXT, error TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, last_delivered_at TEXT,
|
||||
observation_success INTEGER NOT NULL DEFAULT 0,
|
||||
routine INTEGER NOT NULL DEFAULT 0, server_name TEXT, target_path TEXT, source TEXT,
|
||||
UNIQUE(host_id, idempotency_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ec_host_operations_queue ON ec_host_operations(host_id, state, created_at);
|
||||
@@ -93,6 +94,21 @@ export function ensureHostSchema(handle: Database.Database): void {
|
||||
}
|
||||
handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_observation_retention ON ec_host_operations(updated_at)
|
||||
WHERE observation_success=1 AND state='completed'`);
|
||||
if (!columns.some(column => column.name === 'routine')) {
|
||||
handle.transaction(() => {
|
||||
handle.exec(`ALTER TABLE ec_host_operations ADD COLUMN routine INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE ec_host_operations ADD COLUMN server_name TEXT;
|
||||
ALTER TABLE ec_host_operations ADD COLUMN target_path TEXT;
|
||||
ALTER TABLE ec_host_operations ADD COLUMN source TEXT;`);
|
||||
const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as { name: string }[];
|
||||
const nativeInstances = ['id', 'server'].every(name => nativeColumns.some(column => column.name === name));
|
||||
handle.exec(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
|
||||
target_path=${operationTargetSql}, server_name=${nativeInstances ? '(SELECT n.server FROM native_instances n WHERE n.id=o.instance_id)' : 'NULL'}`);
|
||||
})();
|
||||
}
|
||||
handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_routine_history ON ec_host_operations(routine,created_at);
|
||||
CREATE INDEX IF NOT EXISTS ec_host_operations_important_history ON ec_host_operations(created_at)
|
||||
WHERE ${importantOperationSql.replaceAll('o.', '')}`);
|
||||
const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
|
||||
if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
|
||||
const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[];
|
||||
|
||||
@@ -10,6 +10,7 @@ import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '
|
||||
import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
|
||||
import { deploymentStage, finishDeployment } from '../provisioning/history.js';
|
||||
import { recordProgress } from '../provisioning/progress.js';
|
||||
import { withOperationSource } from '../hosts/operation-context.js';
|
||||
import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js';
|
||||
|
||||
import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
|
||||
@@ -309,7 +310,8 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
|
||||
let inFlight: Promise<void> | null = null;
|
||||
export function watchdogTick(forceScan = false): Promise<void> {
|
||||
if (inFlight) return inFlight;
|
||||
inFlight = run(forceScan).finally(() => { inFlight = null; });
|
||||
inFlight = withOperationSource(forceScan ? 'Manual maintenance check' : 'Maintenance watchdog', () => run(forceScan))
|
||||
.finally(() => { inFlight = null; });
|
||||
return inFlight;
|
||||
}
|
||||
async function run(forceScan: boolean): Promise<void> {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { WorkerActivity } from '../../provisioning-detail.js';
|
||||
import { currentOperationSource, withOperationSource } from '../hosts/operation-context.js';
|
||||
|
||||
const active = new Map<string, WorkerActivity>();
|
||||
let sequence = 0;
|
||||
@@ -11,6 +12,6 @@ export function workerActivity(): WorkerActivity[] {
|
||||
export async function trackActivity<T>(key: string, label: string, work: () => Promise<T>): Promise<T> {
|
||||
if (active.has(key)) key = `${key}:${++sequence}`;
|
||||
active.set(key, { key, label, startedAt: new Date().toISOString() });
|
||||
try { return await work(); }
|
||||
try { return await withOperationSource(currentOperationSource() ?? label, work); }
|
||||
finally { active.delete(key); }
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import { selectedZombieMap } from '../../map-aliases.js';
|
||||
import { runMaintenance } from '../maintenance/index.js';
|
||||
import { cleanupControllerLibraries } from '../game-files/index.js';
|
||||
import { trackActivity } from './activity.js';
|
||||
import { withOperationSource } from '../hosts/operation-context.js';
|
||||
|
||||
const TICK_MS = 15_000;
|
||||
// Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
|
||||
@@ -104,7 +105,7 @@ function queueStatus(server: string): Promise<number | null> {
|
||||
statusQueue.push(async () => {
|
||||
let capacity: number | null = null;
|
||||
try {
|
||||
capacity = await refreshServerStatus(server);
|
||||
capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server));
|
||||
// Publish this server's latest evidence without waiting for the fleet.
|
||||
const current = getServer(server);
|
||||
if (current) publishStatus(statusEntry(current));
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
import { error, json } from '@sveltejs/kit';
|
||||
import { listHostOperationHistory } from '$lib/server/hosts/index.js';
|
||||
import { requireManagementOrigin } from '$lib/server/hosts/http.js';
|
||||
import { isHostOperationFilter } from '$lib/hosts.js';
|
||||
|
||||
export function GET({ request }: { request: Request }) {
|
||||
requireManagementOrigin(request);
|
||||
const page = Number(new URL(request.url).searchParams.get('page') ?? '1');
|
||||
const params = new URL(request.url).searchParams;
|
||||
const page = Number(params.get('page') ?? '1');
|
||||
const filter = params.get('filter') ?? 'important';
|
||||
if (!Number.isSafeInteger(page) || page < 1) error(400, 'Invalid operation history page.');
|
||||
return json(listHostOperationHistory(page), { headers: { 'Cache-Control': 'no-store' } });
|
||||
if (!isHostOperationFilter(filter)) error(400, 'Invalid operation history filter.');
|
||||
return json(listHostOperationHistory(page, filter), { headers: { 'Cache-Control': 'no-store' } });
|
||||
}
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
import { redirect } from '@sveltejs/kit';
|
||||
import { listHostOperationHistory } from '$lib/server/hosts/index.js';
|
||||
import { isHostOperationFilter } from '$lib/hosts.js';
|
||||
import type { PageServerLoad } from './$types';
|
||||
|
||||
export const load: PageServerLoad = ({ url }) => {
|
||||
const requestedPage = url.searchParams.get('page');
|
||||
const history = listHostOperationHistory(Number(requestedPage ?? 1));
|
||||
if (requestedPage !== null && requestedPage !== String(history.page)) {
|
||||
redirect(307, history.page === 1 ? '/hosts/operations' : `/hosts/operations?page=${history.page}`);
|
||||
const requestedFilter = url.searchParams.get('filter');
|
||||
const filter = isHostOperationFilter(requestedFilter) ? requestedFilter : 'important';
|
||||
const history = listHostOperationHistory(Number(requestedPage ?? 1), filter);
|
||||
if ((requestedPage !== null && requestedPage !== String(history.page))
|
||||
|| (requestedFilter !== null && requestedFilter !== filter)) {
|
||||
const params = new URLSearchParams();
|
||||
if (requestedFilter !== null) params.set('filter', filter);
|
||||
if (history.page !== 1) params.set('page', String(history.page));
|
||||
redirect(307, `/hosts/operations${params.size ? `?${params}` : ''}`);
|
||||
}
|
||||
return { history };
|
||||
};
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import { onMount, untrack } from 'svelte';
|
||||
import { beforeNavigate } from '$app/navigation';
|
||||
import { navigating } from '$app/stores';
|
||||
import type { HostOperationHistoryPage } from '$lib/hosts.js';
|
||||
import type { HostOperationFilter, HostOperationHistoryPage } from '$lib/hosts.js';
|
||||
import type { PageData } from './$types';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { timeFormat } from '$ui/stores/time-zone.js';
|
||||
@@ -19,7 +19,17 @@
|
||||
const changingPage = $derived($navigating?.to?.url.pathname === '/hosts/operations');
|
||||
const firstOperation = $derived((history.page - 1) * history.pageSize + 1);
|
||||
const numbers = new Intl.NumberFormat('en-US');
|
||||
function pageHref(page: number) { return page === 1 ? '/hosts/operations' : `/hosts/operations?page=${page}`; }
|
||||
const filters: Array<{ id: HostOperationFilter; label: string; description: string; empty: string }> = [
|
||||
{ id: 'important', label: 'Actions & errors', description: 'Actions, failures, and checks that need attention.', empty: 'No actions or checks needing attention recorded.' },
|
||||
{ id: 'routine', label: 'Routine checks', description: 'Monitoring and file reads across all states.', empty: 'No routine checks recorded.' },
|
||||
{ id: 'all', label: 'All operations', description: 'Every recorded remote operation.', empty: 'No remote operations yet.' }
|
||||
];
|
||||
const selectedFilter = $derived(filters.find(filter => filter.id === history.filter)!);
|
||||
function pageHref(page: number, filter: HostOperationFilter = history.filter) {
|
||||
const params = new URLSearchParams({ filter });
|
||||
if (page !== 1) params.set('page', String(page));
|
||||
return `/hosts/operations?${params}`;
|
||||
}
|
||||
function cancel() { sequence++; request?.abort(); request = null; refreshing = false; }
|
||||
|
||||
async function refresh(signal?: AbortSignal) {
|
||||
@@ -30,7 +40,8 @@
|
||||
request = controller;
|
||||
refreshing = true;
|
||||
try {
|
||||
const response = await fetch(`/api/hosts/operations?page=${history.page}`, {
|
||||
const params = new URLSearchParams({ page: String(history.page), filter: history.filter });
|
||||
const response = await fetch(`/api/hosts/operations?${params}`, {
|
||||
signal: AbortSignal.any([controller.signal, AbortSignal.timeout(15000), ...(signal ? [signal] : [])])
|
||||
});
|
||||
const body = await response.json();
|
||||
@@ -55,18 +66,32 @@
|
||||
<SectionHeader title="Operation history" eyebrow="Hosts" subtitle="Remote operations across all hosts, newest first.">
|
||||
{#snippet actions()}<Button disabled={refreshing || changingPage} onclick={() => void refresh()}>{refreshing ? 'Refreshing…' : 'Refresh'}</Button>{/snippet}
|
||||
</SectionHeader>
|
||||
<div class="history-filters">
|
||||
<nav class="filter-links" aria-label="Operation history filters">
|
||||
{#each filters as filter}
|
||||
<a href={pageHref(1, filter.id)} class:active={history.filter === filter.id}
|
||||
aria-current={history.filter === filter.id ? 'page' : undefined}>{filter.label}</a>
|
||||
{/each}
|
||||
</nav>
|
||||
<p>{selectedFilter.description}</p>
|
||||
</div>
|
||||
{#if refreshError}<p class="ui-error" role="alert">{refreshError}</p>{/if}
|
||||
<section class="panel" aria-label="Host operation history">
|
||||
{#if !history.operations.length}<p class="empty" role="status">No remote operations yet.</p>
|
||||
{#if !history.operations.length}<p class="empty" role="status">{selectedFilter.empty}</p>
|
||||
{:else}
|
||||
<!-- svelte-ignore a11y_no_noninteractive_tabindex (Scrollable table for keyboard users.) -->
|
||||
<div class="table-wrap" role="region" aria-label="Operations" tabindex="0">
|
||||
<table>
|
||||
<thead><tr><th scope="col">Operation</th><th scope="col">Host</th><th scope="col">State</th><th scope="col">Updated</th><th scope="col">Result</th></tr></thead>
|
||||
<thead><tr><th scope="col">Operation</th><th scope="col">Host</th><th scope="col">Target</th><th scope="col">Source</th><th scope="col">State</th><th scope="col">Updated</th><th scope="col">Result</th></tr></thead>
|
||||
<tbody>{#each history.operations as operation (operation.id)}
|
||||
<tr>
|
||||
<td>{operation.kind}<small>{operation.id}</small></td>
|
||||
<td class="operation">{operation.kind}<small>{operation.id}</small></td>
|
||||
<td>{#if operation.hostName !== null}<a href={`/hosts/${operation.hostId}`}>{operation.hostName}</a>{:else}Deleted host<small>{operation.hostId}</small>{/if}</td>
|
||||
<td class="target">
|
||||
<span>{operation.serverName ?? operation.instanceId ?? 'Host-level'}</span>
|
||||
{#if operation.targetPath}<small class="target-path">{operation.targetPath}</small>{/if}
|
||||
</td>
|
||||
<td class="source">{operation.source ?? 'Not recorded'}</td>
|
||||
<td class="state" class:failed={operation.state === 'failed'}>{operation.state}</td>
|
||||
<td class="updated">{$timeFormat.dateTime(operation.updatedAt)}</td>
|
||||
<td class="result">{operation.error ?? '—'}</td>
|
||||
@@ -88,14 +113,25 @@
|
||||
|
||||
<style>
|
||||
.page { display: grid; gap: 24px; min-width: 0; }
|
||||
.history-filters { min-width: 0; }
|
||||
.history-filters p { margin: 10px 0 0; color: var(--text-secondary); font-size: var(--text-sm); }
|
||||
.filter-links { display: flex; gap: 4px; width: fit-content; max-width: 100%; padding: 4px; overflow-x: auto; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); }
|
||||
.filter-links a { padding: 8px 16px; border-radius: var(--radius-sm); color: var(--text-secondary); font-size: var(--text-sm); text-decoration: none; white-space: nowrap; }
|
||||
.filter-links a:hover { color: var(--text-primary); }
|
||||
.filter-links a.active { color: var(--text-primary); background: var(--surface-raised); }
|
||||
.filter-links a:focus-visible { outline: 2px solid var(--accent-primary); outline-offset: -2px; }
|
||||
.panel { min-width: 0; border: 1px solid var(--border-strong); border-radius: var(--radius-md); background: var(--surface); overflow: hidden; }
|
||||
.table-wrap { overflow: auto; }
|
||||
.table-wrap:focus-visible { outline: 2px solid var(--accent-primary); outline-offset: -2px; }
|
||||
table { width: 100%; min-width: 760px; border-collapse: collapse; text-align: left; font-size: var(--text-sm); }
|
||||
table { width: 100%; min-width: 1050px; border-collapse: collapse; text-align: left; font-size: var(--text-sm); }
|
||||
td, th { padding: 16px; border-bottom: 1px solid var(--border-subtle); vertical-align: top; }
|
||||
th { color: var(--text-secondary); font-size: var(--text-xs); font-weight: var(--weight-semibold); }
|
||||
td { line-height: 1.6; overflow-wrap: anywhere; }
|
||||
small { display: block; color: var(--text-secondary); font-size: var(--text-xs); margin-top: 4px; }
|
||||
.operation { min-width: 180px; max-width: 240px; }
|
||||
.target { min-width: 180px; max-width: 320px; }
|
||||
.target-path { font-family: var(--font-mono); }
|
||||
.source { min-width: 120px; max-width: 220px; }
|
||||
.state { text-transform: capitalize; }
|
||||
.failed { color: var(--state-danger); }
|
||||
.updated { white-space: nowrap; }
|
||||
@@ -103,5 +139,9 @@
|
||||
.pagination { padding: 16px; display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 16px; color: var(--text-secondary); font-size: var(--text-xs); }
|
||||
.page-controls { display: flex; align-items: center; gap: 12px; }
|
||||
.empty { padding: 24px; margin: 0; color: var(--text-secondary); font-size: var(--text-sm); }
|
||||
@media (max-width: 600px) { .page-controls { width: 100%; justify-content: space-between; } }
|
||||
@media (max-width: 600px) {
|
||||
td, th { padding: 12px; }
|
||||
.page-controls { width: 100%; justify-content: space-between; }
|
||||
.filter-links a { padding: 8px 12px; }
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -123,6 +123,7 @@ describe('routine host observation retention', () => {
|
||||
api.handle.prepare('UPDATE ec_host_operations SET result=? WHERE id=?').run('{not-json', malformed);
|
||||
retained.push(malformed);
|
||||
api.handle.exec('DROP INDEX ec_host_operations_observation_retention');
|
||||
api.handle.exec('DROP INDEX ec_host_operations_important_history');
|
||||
api.handle.exec('ALTER TABLE ec_host_operations DROP COLUMN observation_success');
|
||||
api.closeDb();
|
||||
|
||||
|
||||
@@ -30,27 +30,43 @@ async function seedHistory(count = 53) {
|
||||
return { ...hosts, handle, ids, first, second, deleted };
|
||||
}
|
||||
|
||||
async function operationHistory() {
|
||||
const api = await seedHistory(0);
|
||||
function add(kind: string, options: { payload?: Record<string, unknown>; state?: string; result?: unknown; error?: string } = {}) {
|
||||
const operation = api.queueHostOperation(api.first.id, kind, options.payload ?? {});
|
||||
if (options.state !== 'pending') api.reportHostOperationStatus(api.first.id, operation.id, {
|
||||
state: options.state ?? 'completed', result: options.result, error: options.error
|
||||
});
|
||||
return operation.id;
|
||||
}
|
||||
return { ...api, add };
|
||||
}
|
||||
|
||||
describe('host operation history', () => {
|
||||
it('pages all hosts in stable newest-first order, retaining deleted-host records', async () => {
|
||||
const api = await seedHistory();
|
||||
const pages = [1, 2, 3].map(page => api.listHostOperationHistory(page));
|
||||
const pages = [1, 2, 3].map(page => api.listHostOperationHistory(page, 'all'));
|
||||
expect(pages.map(page => page.operations.length)).toEqual([25, 25, 3]);
|
||||
pages.forEach((page, index) => expect(page).toMatchObject({ page: index + 1, pageSize: 25, total: 53, totalPages: 3 }));
|
||||
pages.forEach((page, index) => expect(page).toMatchObject({ page: index + 1, pageSize: 25, total: 53, totalPages: 3, filter: 'all' }));
|
||||
const operations = pages.flatMap(page => page.operations);
|
||||
expect(operations.map(operation => operation.id)).toEqual([...api.ids].reverse());
|
||||
expect(operations.find(operation => operation.hostId === api.first.id)?.hostName).toBe('Alpha host');
|
||||
expect(operations.find(operation => operation.hostId === api.second.id)?.hostName).toBe('Beta host');
|
||||
expect(operations.filter(operation => operation.hostId === api.deleted.id).length).toBeGreaterThan(0);
|
||||
expect(operations.filter(operation => operation.hostId === api.deleted.id).every(operation => operation.hostName === null)).toBe(true);
|
||||
expect(api.listHostOperationHistory(999)).toEqual(pages[2]);
|
||||
expect(api.listHostOperationHistory(999, 'all')).toEqual(pages[2]);
|
||||
});
|
||||
|
||||
it('bounds reads in SQLite using the history index and omits private payloads and results', async () => {
|
||||
const api = await seedHistory(1003);
|
||||
const prepare = vi.spyOn(api.handle, 'prepare');
|
||||
const history = api.listHostOperationHistory(2);
|
||||
const history = api.listHostOperationHistory(2, 'all');
|
||||
api.listHostOperationHistory(1, 'important');
|
||||
api.listHostOperationHistory(1, 'routine');
|
||||
const statements = prepare.mock.calls.map(([sql]) => sql);
|
||||
const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o LEFT JOIN'))!;
|
||||
prepare.mockRestore();
|
||||
expect(statements.every(sql => !/\b(?:payload|result|json_extract|ec_host_operation_payloads)\b/.test(sql))).toBe(true);
|
||||
const projection = query.split('FROM ec_host_operations')[0];
|
||||
expect(projection).not.toContain('payload');
|
||||
expect(projection).not.toContain('result');
|
||||
@@ -65,28 +81,167 @@ describe('host operation history', () => {
|
||||
expect(JSON.stringify(history)).not.toContain('secret');
|
||||
});
|
||||
|
||||
it('seeks through the selected history index without sorting or reading private payloads', async () => {
|
||||
const api = await seedHistory(1003);
|
||||
for (const filter of ['important', 'routine'] as const) {
|
||||
const prepare = vi.spyOn(api.handle, 'prepare');
|
||||
api.listHostOperationHistory(2, filter);
|
||||
const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o LEFT JOIN'))!;
|
||||
prepare.mockRestore();
|
||||
const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(25, 25) as { detail: string }[];
|
||||
expect(plan.some(row => row.detail.includes(`USING INDEX ec_host_operations_${filter}_history`))).toBe(true);
|
||||
expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false);
|
||||
expect(query).not.toMatch(/\b(?:payload|result|json_extract)\b/);
|
||||
}
|
||||
});
|
||||
|
||||
it('returns an empty first page and normalizes invalid internal page numbers', async () => {
|
||||
const api = await seedHistory(0);
|
||||
const empty = { operations: [], page: 1, pageSize: 25, total: 0, totalPages: 1 };
|
||||
const empty = { operations: [], page: 1, pageSize: 25, total: 0, totalPages: 1, filter: 'important' };
|
||||
expect(api.listHostOperationHistory()).toEqual(empty);
|
||||
for (const page of [0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, 999]) {
|
||||
expect(api.listHostOperationHistory(page)).toEqual(empty);
|
||||
}
|
||||
});
|
||||
|
||||
it('defaults to actions and errors, with independent counts and pagination for every filter', async () => {
|
||||
const api = await operationHistory();
|
||||
const actions = Array.from({ length: 26 }, () => api.add('instance.restart'));
|
||||
actions.push(api.add('rcon.execute', { payload: { command: 'map_restart' } }));
|
||||
const successful = Array.from({ length: 26 }, () => api.add('files.manifest', { result: { files: [] } }));
|
||||
const pending = ['pending', 'accepted', 'running'].map(state => api.add('files.read', { state }));
|
||||
const failures = ['failed', 'unknown'].map(state => api.add('files.read', { state, error: 'Observation unavailable' }));
|
||||
const negative = [
|
||||
api.add('rcon.execute', { payload: { command: 'status' }, result: { ok: false, reason: 'timeout' } }),
|
||||
api.add('instance.inspect', { result: { healthy: false, state: 'stopped' } }),
|
||||
api.add('files.read', { result: { error: 'Read was incomplete' } }),
|
||||
api.add('files.stat', { error: 'Could not inspect the file' })
|
||||
];
|
||||
const ids = [...actions, ...successful, ...pending, ...failures, ...negative];
|
||||
for (const [filter, matching] of [
|
||||
['important', [...actions, ...failures, ...negative]],
|
||||
['routine', [...successful, ...pending, ...failures, ...negative]],
|
||||
['all', ids]
|
||||
] as const) {
|
||||
const pages = Array.from({ length: Math.ceil(matching.length / 25) }, (_, index) => api.listHostOperationHistory(index + 1, filter));
|
||||
expect(pages[0]).toMatchObject({ filter, total: matching.length, totalPages: pages.length, pageSize: 25 });
|
||||
expect(pages.flatMap(page => page.operations.map(operation => operation.id))).toEqual([...matching].reverse());
|
||||
expect(api.listHostOperationHistory(999, filter)).toEqual(pages.at(-1));
|
||||
}
|
||||
expect(api.listHostOperationHistory()).toEqual(api.listHostOperationHistory(1, 'important'));
|
||||
const routine = api.listHostOperationHistory(1, 'routine').operations;
|
||||
expect(routine.every(operation => operation.routine)).toBe(true);
|
||||
expect(api.listHostOperationHistory(2, 'important').operations.every(operation => !operation.routine)).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps manual RCON commands visible and only exposes safe read command names', async () => {
|
||||
const api = await operationHistory();
|
||||
const reads = ['status', 'getstatus', 'version'].map(command => api.add('rcon.execute', { payload: { command } }));
|
||||
const manual = api.add('rcon.execute', { payload: { command: 'say credential-secret' } });
|
||||
const combined = api.add('rcon.execute', { payload: { command: 'status; map_restart' } });
|
||||
const important = api.listHostOperationHistory();
|
||||
expect(important.operations.map(operation => operation.id)).toEqual([combined, manual]);
|
||||
expect(important.operations.every(operation => !operation.routine && operation.targetPath === null)).toBe(true);
|
||||
const routine = api.listHostOperationHistory(1, 'routine');
|
||||
expect(routine.operations.map(operation => operation.id)).toEqual([...reads].reverse());
|
||||
expect(routine.operations.map(operation => operation.targetPath)).toEqual(['version', 'getstatus', 'status']);
|
||||
expect(JSON.stringify(api.listHostOperationHistory(1, 'all'))).not.toContain('credential-secret');
|
||||
});
|
||||
|
||||
it('persists server, target, and source summaries without depending on current instance records', async () => {
|
||||
const api = await operationHistory();
|
||||
const { withOperationSource } = await import('../../src/lib/server/hosts/operation-context.js');
|
||||
api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES('history-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)`)
|
||||
.run(api.first.id, new Date().toISOString());
|
||||
const read = withOperationSource('Routine status', () => api.queueHostOperation(api.first.id, 'files.read', {
|
||||
path: 'configs/server.cfg', credential: 'payload-secret'
|
||||
}, { instanceId: 'history-instance', generation: 1 }));
|
||||
const manifest = withOperationSource('Watchdog drift scan', () => api.queueHostOperation(api.first.id, 'files.manifest', {
|
||||
roots: ['scripts', 'configs', 'plugins'], ignoreNames: [], paths: ['configs/server.cfg']
|
||||
}, { instanceId: 'history-instance', generation: 1 }));
|
||||
api.reportHostOperationStatus(api.first.id, read.id, { state: 'completed', result: { contentBase64: 'result-secret' } });
|
||||
api.handle.prepare('DELETE FROM native_instances WHERE id=?').run('history-instance');
|
||||
const history = api.listHostOperationHistory(1, 'all');
|
||||
expect(history.operations.find(operation => operation.id === read.id)).toMatchObject({
|
||||
serverName: 't5_zombie_theater_1', targetPath: 'configs/server.cfg', source: 'Routine status', routine: true
|
||||
});
|
||||
expect(history.operations.find(operation => operation.id === manifest.id)).toMatchObject({
|
||||
serverName: 't5_zombie_theater_1', targetPath: 'scripts, configs, plugins', source: 'Watchdog drift scan', routine: true
|
||||
});
|
||||
expect(JSON.stringify(history)).not.toContain('secret');
|
||||
});
|
||||
|
||||
it('refuses unsafe target summaries and never derives labels from arbitrary payload fields', async () => {
|
||||
const api = await operationHistory();
|
||||
const unsafe = ['/etc/private-secret', '../private-secret', 'configs/../../private-secret', 'configs/line\nprivate-secret', 'configs\\private-secret'];
|
||||
const ids = unsafe.map(path => api.add('files.read', { payload: { path, command: 'command-secret', name: 'name-secret' } }));
|
||||
ids.push(api.add('files.manifest', { payload: { roots: ['scripts', '../private-secret'] } }));
|
||||
ids.push(api.add('instance.restart', { payload: { path: 'configs/private-secret', command: 'command-secret' } }));
|
||||
const history = api.listHostOperationHistory(1, 'all');
|
||||
expect(history.operations).toHaveLength(ids.length);
|
||||
expect(history.operations.every(operation => operation.targetPath === null && operation.serverName === null)).toBe(true);
|
||||
expect(JSON.stringify(history)).not.toContain('secret');
|
||||
});
|
||||
|
||||
it('backfills safe legacy summaries from immutable payloads while leaving source unknown', async () => {
|
||||
const api = await operationHistory();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES('legacy-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)`)
|
||||
.run(api.first.id, new Date().toISOString());
|
||||
const read = api.queueHostOperationSerialized(api.first.id, 'files.read', Buffer.from(JSON.stringify({
|
||||
path: 'configs/server.cfg', credential: 'payload-secret'
|
||||
})), { instanceId: 'legacy-instance', generation: 1 }, { path: 'configs/routing-secret' });
|
||||
api.reportHostOperationStatus(api.first.id, read.id, { state: 'completed', result: { credential: 'result-secret' } });
|
||||
const command = api.queueHostOperationSerialized(api.first.id, 'rcon.execute', Buffer.from('{"command":"status"}'), {}, { command: 'command-secret' });
|
||||
const manual = api.add('rcon.execute', { payload: { command: 'say command-secret' } });
|
||||
const unsafe = api.add('files.read', { payload: { path: '../private-secret' } });
|
||||
api.handle.exec(`DROP INDEX ec_host_operations_routine_history;
|
||||
DROP INDEX ec_host_operations_important_history;
|
||||
ALTER TABLE ec_host_operations DROP COLUMN routine;
|
||||
ALTER TABLE ec_host_operations DROP COLUMN server_name;
|
||||
ALTER TABLE ec_host_operations DROP COLUMN target_path;
|
||||
ALTER TABLE ec_host_operations DROP COLUMN source;`);
|
||||
closeDb();
|
||||
const history = api.listHostOperationHistory(1, 'all');
|
||||
expect(history.operations).toHaveLength(4);
|
||||
expect(history.operations.find(operation => operation.id === read.id)).toMatchObject({
|
||||
routine: true, serverName: 't5_zombie_theater_1', targetPath: 'configs/server.cfg', source: null
|
||||
});
|
||||
expect(history.operations.find(operation => operation.id === command.id)).toMatchObject({ routine: true, targetPath: 'status', source: null });
|
||||
expect(history.operations.find(operation => operation.id === manual)).toMatchObject({ routine: false, targetPath: null, source: null });
|
||||
expect(history.operations.find(operation => operation.id === unsafe)).toMatchObject({ routine: true, targetPath: null, source: null });
|
||||
expect(JSON.stringify(history)).not.toContain('secret');
|
||||
closeDb();
|
||||
expect(api.listHostOperationHistory(1, 'all')).toEqual(history);
|
||||
});
|
||||
|
||||
it('serves fixed-size API pages without caching or exposing private operation data', async () => {
|
||||
const api = await seedHistory();
|
||||
const { GET } = await import('../../src/routes/api/hosts/operations/+server.js');
|
||||
const first = GET({ request: new Request('https://ec.test/api/hosts/operations') });
|
||||
const first = GET({ request: new Request('https://ec.test/api/hosts/operations?filter=all') });
|
||||
expect(first.headers.get('cache-control')).toBe('no-store');
|
||||
expect(await first.json()).toEqual(api.listHostOperationHistory(1));
|
||||
const second = GET({ request: new Request('https://ec.test/api/hosts/operations?page=2&pageSize=500') });
|
||||
expect(await first.json()).toEqual(api.listHostOperationHistory(1, 'all'));
|
||||
const second = GET({ request: new Request('https://ec.test/api/hosts/operations?page=2&pageSize=500&filter=all') });
|
||||
const body = await second.json();
|
||||
expect(body).toEqual(api.listHostOperationHistory(2));
|
||||
expect(body).toEqual(api.listHostOperationHistory(2, 'all'));
|
||||
expect(body.operations).toHaveLength(25);
|
||||
expect(JSON.stringify(body)).not.toContain('secret');
|
||||
const last = GET({ request: new Request('https://ec.test/api/hosts/operations?page=999') });
|
||||
expect(await last.json()).toEqual(api.listHostOperationHistory(3));
|
||||
const last = GET({ request: new Request('https://ec.test/api/hosts/operations?page=999&filter=all') });
|
||||
expect(await last.json()).toEqual(api.listHostOperationHistory(3, 'all'));
|
||||
});
|
||||
|
||||
it('serves the selected filter and defaults the API to actions and errors', async () => {
|
||||
const api = await operationHistory();
|
||||
const action = api.add('instance.restart');
|
||||
const read = api.add('instance.inspect', { result: { healthy: true } });
|
||||
const { GET } = await import('../../src/routes/api/hosts/operations/+server.js');
|
||||
const request = (query = '') => GET({ request: new Request(`https://ec.test/api/hosts/operations${query}`) }).json();
|
||||
expect(await request()).toEqual(api.listHostOperationHistory());
|
||||
expect((await request()).operations.map((operation: { id: string }) => operation.id)).toEqual([action]);
|
||||
expect((await request('?filter=routine')).operations.map((operation: { id: string }) => operation.id)).toEqual([read]);
|
||||
expect(await request('?filter=all')).toEqual(api.listHostOperationHistory(1, 'all'));
|
||||
});
|
||||
|
||||
it('rejects invalid API pages and keeps host management access restrictions', async () => {
|
||||
@@ -95,6 +250,9 @@ describe('host operation history', () => {
|
||||
for (const page of ['', '0', '-1', '1.5', 'nope', 'Infinity', '9007199254740992']) {
|
||||
expect(() => GET({ request: new Request(`${url}?page=${page}`) })).toThrow(expect.objectContaining({ status: 400 }));
|
||||
}
|
||||
for (const filter of ['', 'unknown', 'ALL', 'important%27%20OR%201=1']) {
|
||||
expect(() => GET({ request: new Request(`${url}?filter=${filter}`) })).toThrow(expect.objectContaining({ status: 400 }));
|
||||
}
|
||||
const blockedHeaders: HeadersInit[] = [
|
||||
{ origin: 'https://other.test' },
|
||||
{ 'sec-fetch-site': 'cross-site' },
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { currentOperationSource, withOperationSource } from '../../src/lib/server/hosts/operation-context.js';
|
||||
import { trackActivity, workerActivity } from '../../src/lib/server/pipeline/activity.js';
|
||||
|
||||
describe('host operation source context', () => {
|
||||
it('isolates overlapping asynchronous sources and restores the caller', async () => {
|
||||
let release!: () => void;
|
||||
const barrier = new Promise<void>(resolve => { release = resolve; });
|
||||
const first = withOperationSource('GET /servers/[name]', async () => {
|
||||
await barrier;
|
||||
return currentOperationSource();
|
||||
});
|
||||
const second = withOperationSource('Status monitor', async () => {
|
||||
release();
|
||||
await Promise.resolve();
|
||||
return currentOperationSource();
|
||||
});
|
||||
expect(currentOperationSource()).toBeNull();
|
||||
expect(await Promise.all([first, second])).toEqual(['GET /servers/[name]', 'Status monitor']);
|
||||
expect(currentOperationSource()).toBeNull();
|
||||
});
|
||||
|
||||
it('lets explicit background work override incidental request context without leaking it', async () => {
|
||||
await withOperationSource('POST /api/servers/[name]/refresh', async () => {
|
||||
await withOperationSource('Status monitor', async () => {
|
||||
await Promise.resolve();
|
||||
expect(currentOperationSource()).toBe('Status monitor');
|
||||
});
|
||||
expect(currentOperationSource()).toBe('POST /api/servers/[name]/refresh');
|
||||
expect(() => withOperationSource('Failing task', () => { throw new Error('failed'); })).toThrow('failed');
|
||||
expect(currentOperationSource()).toBe('POST /api/servers/[name]/refresh');
|
||||
});
|
||||
expect(currentOperationSource()).toBeNull();
|
||||
});
|
||||
|
||||
it('gives background activity a source while preserving its initiating source through nested activity', async () => {
|
||||
await trackActivity('context-test', 'Reconciling campaigns', async () => {
|
||||
await Promise.resolve();
|
||||
expect(currentOperationSource()).toBe('Reconciling campaigns');
|
||||
await trackActivity('context-test-child', 'Waiting for files.read', async () => {
|
||||
expect(currentOperationSource()).toBe('Reconciling campaigns');
|
||||
});
|
||||
});
|
||||
await withOperationSource('POST /api/servers/[name]/restart', () =>
|
||||
trackActivity('context-test-request', 'Processing provisioning requests', async () => {
|
||||
expect(currentOperationSource()).toBe('POST /api/servers/[name]/restart');
|
||||
}));
|
||||
expect(currentOperationSource()).toBeNull();
|
||||
expect(workerActivity()).toEqual([]);
|
||||
});
|
||||
|
||||
it('restores context and clears activity when asynchronous work rejects', async () => {
|
||||
await withOperationSource('Manual maintenance check', async () => {
|
||||
await expect(trackActivity('context-test-error', 'Waiting for files.read', async () => {
|
||||
await Promise.resolve();
|
||||
expect(currentOperationSource()).toBe('Manual maintenance check');
|
||||
throw new Error('unavailable');
|
||||
})).rejects.toThrow('unavailable');
|
||||
expect(currentOperationSource()).toBe('Manual maintenance check');
|
||||
});
|
||||
expect(currentOperationSource()).toBeNull();
|
||||
expect(workerActivity()).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user