josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Unify retained operation history in Activity

Browse Source
This commit is contained in:
josh
2026-10-07 19:30:38 -04:00
parent c39563daef
commit 05bdeaf624
35 changed files with 1008 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/activity.ts
+188
View File
@@ -0,0 +1,188 @@
import type Database from 'better-sqlite3';
import { normalizeActivityQuery, type ActivityPage, type ActivityQuery, type ActivityRow, type ActivitySource } from '../activity.js';
import { serverTitle } from '../ui/server-labels.js';
import { db } from './db/index.js';
import { ensureHostSchema } from './hosts/schema.js';
import { importantOperationSql } from './hosts/observations.js';
const initialized = new WeakSet<Database.Database>();
const quote = (value: string) => `'${value.replaceAll("'", "''")}'`;
const hostTitles: Record<string, string> = {
'instance.create': 'Create server container', 'instance.rebuild': 'Rebuild server container',
'instance.start': 'Start server container', 'instance.restart': 'Restart server container',
'instance.stop': 'Stop server container', 'instance.remove': 'Remove server container',
'instance.cancel-create': 'Cancel server creation', 'instance.inspect': 'Check server container',
'host.inspect': 'Check host', 'host.decommission': 'Remove host', 'runtime.pull': 'Prepare container runtime',
'library.download': 'Download game files', 'library.import': 'Import game files',
'plutonium.prepare': 'Prepare game update', 'files.stage': 'Stage server files',
'files.apply': 'Apply server files', 'files.read': 'Read server files', 'files.list': 'List server files',
'files.stat': 'Check server files', 'files.manifest': 'Check file versions', 'files.remove': 'Remove server files',
'rcon.status-batch': 'Check game status', 'rcon.execute': 'Game command', 'logs.read': 'Read server logs'
};
const labelCase = (column: string, labels: Record<string, string>, fallback: string) =>
`CASE ${column} ${Object.entries(labels).map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`).join(' ')} ELSE ${quote(fallback)} END`;
// Old attempts may no longer be the native_provisioning pointer. Their stable
// idempotency keys still identify the same retained controller operation.
const lifecycleKey = `CASE
WHEN substr(o.idempotency_key,-6)=':stage' THEN substr(o.idempotency_key,1,length(o.idempotency_key)-6)
${[':migration-create:', ':migrate-remove:'].map(marker => `WHEN instr(o.idempotency_key,${quote(marker)})>0
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length})<>''
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length}) NOT GLOB '*[^0-9]*'
THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`).join('\n')}
ELSE o.idempotency_key END`;
/** One row per user-level operation or finding; no command payloads or observations are read. */
const activitySql = `WITH
links AS (
SELECT operation_id,kind,source_id,server FROM provisioning_sources
UNION SELECT id,substr(triggered_by,1,instr(triggered_by,':')-1),substr(triggered_by,instr(triggered_by,':')+1),server
FROM server_deployments WHERE substr(triggered_by,1,instr(triggered_by,':')-1) IN ('maintenance','watchdog','campaign','beta','autoscaling')
UNION SELECT provisioning_id,'maintenance',CAST(id AS TEXT),server FROM maintenance_jobs WHERE provisioning_id IS NOT NULL
UNION SELECT provisioning_id,'campaign',deployment_id,server FROM deployment_servers WHERE provisioning_id IS NOT NULL
UNION SELECT provisioning_id,'beta',CAST(beta_id AS TEXT),server FROM beta_servers WHERE provisioning_id IS NOT NULL
UNION SELECT a.operation_id,'watchdog',i.id,i.server FROM watchdog_actions a JOIN watchdog_incidents i ON i.id=a.incident_id
),
memberships AS (
SELECT operation_id,'|' || replace(group_concat(DISTINCT CASE kind WHEN 'watchdog' THEN 'recovery' ELSE kind END),',','|') || '|' AS sources
FROM links GROUP BY operation_id
),
maintenance_links AS (
SELECT DISTINCT l.operation_id,m.id,m.state,m.kind,m.target_revision
FROM links l JOIN maintenance_jobs m ON l.kind='maintenance' AND l.source_id=CAST(m.id AS TEXT) AND l.server=m.server
JOIN server_deployments d ON d.id=l.operation_id
WHERE m.provisioning_id=l.operation_id OR (m.provisioning_id IS NULL AND NOT EXISTS (
SELECT 1 FROM links newer JOIN server_deployments n ON n.id=newer.operation_id
WHERE newer.kind='maintenance' AND newer.source_id=l.source_id AND newer.server=l.server
AND (n.started_at>d.started_at OR (n.started_at=d.started_at AND n.rowid>d.rowid))))
),
maintenance_results AS (
SELECT operation_id,MAX(state='failed') AS failed,MAX(state IN ('queued','restarting')) AS pending,
MAX(state='cancelled') AS cancelled,MAX(kind='update') AS is_update,MAX(target_revision) AS revision
FROM maintenance_links GROUP BY operation_id
),
covered_host AS (
SELECT n.operation_id AS id FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.operation_id IS NOT NULL
UNION SELECT n.stage_operation_id FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.stage_operation_id IS NOT NULL
UNION SELECT m.remove_operation_id FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.remove_operation_id IS NOT NULL
),
items AS (
SELECT 'provisioning:' || d.id AS id,d.started_at AS at,
CASE WHEN m.is_update=1 THEN 'update' WHEN d.action='destroy' THEN 'remove' ELSE d.action END AS type,
COALESCE(ms.sources,'|provisioning|') AS sources,
CASE WHEN d.state IN ('failed','interrupted','cancelled','warning') THEN d.state
WHEN m.failed=1 THEN 'failed' WHEN m.pending=1 AND d.state='done' THEN 'verifying'
WHEN m.cancelled=1 THEN 'cancelled' ELSE d.state END AS rawState,
CASE WHEN m.is_update=1 THEN 'Update game' ELSE ${labelCase('d.action', { create: 'Create server', start: 'Start server', restart: 'Restart server', rebuild: 'Rebuild server', stop: 'Stop server', destroy: 'Remove server' }, 'Server operation')} END AS title,
CASE WHEN m.failed=1 AND d.state='done' THEN 'The restart completed, but maintenance verification failed.'
WHEN m.pending=1 AND d.state='done' THEN 'The restart completed. Maintenance verification is still in progress.'
WHEN m.revision IS NOT NULL THEN 'Target game revision r' || m.revision || '.'
WHEN d.state='waiting' THEN 'Waiting for eligibility and available capacity.'
WHEN d.state='awaiting-health' THEN 'Checking that the game is responding.' ELSE NULL END AS detail,
d.server,d.server_deleted_at AS serverDeletedAt,COALESCE(n.host_id,mg.source_host_id) AS hostId,mg.source_host_id AS secondaryHostId,
'provisioning' AS route,d.id AS recordId
FROM server_deployments d LEFT JOIN memberships ms ON ms.operation_id=d.id
LEFT JOIN maintenance_results m ON m.operation_id=d.id
LEFT JOIN native_provisioning p ON p.deployment_id=d.id LEFT JOIN native_instances n ON n.id=p.instance_id
LEFT JOIN ec_host_migrations mg ON mg.deployment_id=d.id
UNION ALL
SELECT 'maintenance:' || m.id,m.requested_at,CASE WHEN m.kind='update' THEN 'update' ELSE 'restart' END,
'|maintenance|',m.state,CASE WHEN m.kind='update' THEN 'Update game' WHEN m.kind='schedule' THEN 'Scheduled server maintenance' ELSE 'Server maintenance' END,
CASE WHEN m.target_revision IS NOT NULL THEN 'Target game revision r' || m.target_revision || '.' ELSE NULL END,
m.server,m.server_deleted_at,NULL,NULL,'maintenance-job',CAST(m.id AS TEXT)
FROM maintenance_jobs m WHERE NOT EXISTS (SELECT 1 FROM maintenance_links l WHERE l.id=m.id)
UNION ALL
SELECT 'recovery:' || i.id,i.first_seen_at,i.kind,'|recovery|',i.state,
CASE WHEN i.kind='health' THEN 'Game health finding' ELSE 'Server files differ from the promoted setup' END,
CASE WHEN i.state='recovering' THEN 'Recovery is in progress.' WHEN i.state='attention' THEN 'Automatic recovery needs review.'
WHEN i.state='resolved' THEN 'The finding was resolved.' WHEN i.state='dismissed' THEN 'The finding was dismissed.' ELSE 'A check found an issue that needs review.' END,
i.server,i.server_deleted_at,NULL,NULL,'recovery',i.id
FROM watchdog_incidents i
UNION ALL
SELECT 'campaign:' || d.id || ':' || ds.server,d.started_at,CASE WHEN d.action='drift' THEN 'drift' ELSE 'files' END,
CASE WHEN d.triggered_by='watchdog' OR EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=d.id)
THEN '|campaign|recovery|' ELSE '|campaign|' END,ds.restart_state,
CASE d.action WHEN 'promote' THEN 'Promote files' WHEN 'rollback' THEN 'Roll back files' ELSE 'Reconcile files' END,
CAST(ds.files_changed AS TEXT) || CASE WHEN ds.files_changed=1 THEN ' file changed.' ELSE ' files changed.' END,
ds.server,ds.server_deleted_at,NULL,NULL,'campaign',d.id
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id=d.id
UNION ALL
SELECT 'beta:' || b.id || ':' || bs.server,b.opened_at,'beta','|beta|',
CASE WHEN b.state IN ('promoted','rolled-back','cancelled') THEN b.state ELSE bs.stage END,
'Beta test #' || b.id,NULL,bs.server,bs.server_deleted_at,NULL,NULL,'beta',CAST(b.id AS TEXT)
FROM betas b JOIN beta_servers bs ON bs.beta_id=b.id
UNION ALL
SELECT 'host:' || o.id,o.created_at,'host','|host|',
CASE WHEN o.state='completed' AND o.routine=1 AND o.observation_success=0 THEN 'observation-failed'
WHEN o.state='completed' AND COALESCE(o.error,'')<>'' THEN 'warning' ELSE o.state END,
${labelCase('o.kind', hostTitles, 'Host operation')},
CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0 THEN 'The command completed, but the check reported a problem.'
WHEN o.state='unknown' THEN 'The host has not confirmed the result.' ELSE NULL END,
COALESCE(o.server_name,n.server),s.destroyed_at,o.host_id,NULL,'host',o.id
FROM ec_host_operations o LEFT JOIN native_instances n ON n.id=o.instance_id
LEFT JOIN servers s ON s.name=COALESCE(o.server_name,n.server)
WHERE o.history_expired_at IS NULL AND ${importantOperationSql}
AND NOT EXISTS (SELECT 1 FROM covered_host c WHERE c.id=o.id)
AND NOT (o.kind IN ('files.stage','instance.create','instance.rebuild','instance.start','instance.restart','instance.stop','instance.remove','instance.cancel-create')
AND EXISTS (SELECT 1 FROM server_deployments d WHERE d.id=${lifecycleKey}))
),
normalized AS (
SELECT i.*,h.name AS hostName,
CASE WHEN rawState IN ('queued','pending','waiting','held','awaiting-empty','awaiting-match') THEN 'queued'
WHEN rawState IN ('running','accepted','preparing','staging','testing','observing','deploying','restarting','recovering','awaiting-health','verifying','rolling-back') THEN 'running'
WHEN rawState IN ('done','completed','resolved','ready','ready-promote','promoted','rolled-back') THEN 'succeeded'
WHEN rawState IN ('failed','interrupted','observation-failed') THEN 'failed'
WHEN rawState IN ('cancelled','dismissed','skipped') THEN 'cancelled'
WHEN rawState IN ('manual','attention','open') THEN 'attention'
WHEN rawState='warning' THEN 'warning' ELSE 'unknown' END AS state
FROM items i LEFT JOIN ec_hosts h ON h.id=i.hostId
)
`;
type SqlRow = Omit<ActivityRow, 'source' | 'sources' | 'href'> & { sources: string; route: string; recordId: string; secondaryHostId: string | null };
const sourceOrder: ActivitySource[] = ['maintenance', 'recovery', 'campaign', 'beta', 'autoscaling', 'provisioning', 'host'];
function present(row: SqlRow): ActivityRow {
const sources = sourceOrder.filter(source => row.sources.includes(`|${source}|`));
const href = row.route === 'maintenance-job' ? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}`
: row.route === 'recovery' ? `/maintenance/${encodeURIComponent(row.recordId)}`
: row.route === 'host' ? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all`
: `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`;
return { id: row.id, at: row.at, type: row.type, source: sources[0] ?? 'provisioning', sources,
state: row.state, rawState: row.rawState, title: row.title, detail: row.detail,
server: row.server, serverDeletedAt: row.serverDeletedAt, hostId: row.hostId, hostName: row.hostName, href };
}
/** All predicates, totals and pagination operate on the same retained SQL union. */
export function activityHistory(input: ActivityQuery = {}, now = Date.now()): ActivityPage {
const query = normalizeActivityQuery(input);
const handle = db();
ensureHostSchema(handle);
if (!initialized.has(handle)) {
handle.function('activity_server_title', { deterministic: true }, (name: unknown) => typeof name === 'string' ? serverTitle(name) : '');
initialized.add(handle);
}
const conditions: string[] = [];
const bindings: Record<string, string | number> = {};
for (const key of ['server', 'type', 'state'] as const) if (query[key]) { conditions.push(`${key}=@${key}`); bindings[key] = query[key]; }
if (query.host) { conditions.push('(hostId=@host OR secondaryHostId=@host)'); bindings.host = query.host; }
if (query.source) {
// Maintenance is the user-facing umbrella; Recovery remains available as a narrower filter.
conditions.push(query.source === 'maintenance' ? "(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)" : 'instr(sources,@source)>0');
if (query.source !== 'maintenance') bindings.source = `|${query.source}|`;
}
if (query.days) { conditions.push('julianday(at)>=julianday(@since)'); bindings.since = new Date(now - query.days * 86_400_000).toISOString(); }
if (query.q) {
conditions.push("instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0");
bindings.q = query.q.toLocaleLowerCase();
}
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
const limit = query.limit!;
return handle.transaction(() => {
const { total } = handle.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`).get(bindings) as { total: number };
const offset = Math.min(query.offset!, Math.max(0, Math.ceil(total / limit) - 1) * limit);
const rows = handle.prepare(`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId
FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`)
.all({ ...bindings, limit, offset }) as SqlRow[];
return { rows: rows.map(present), total, limit, offset, query: { ...query, offset } };
})();
}
src/lib/server/hosts/index.ts
+6 -2
View File
@@ -438,11 +438,15 @@ export function listHostOperationHistory(page = 1, filter: HostOperationFilter =
const parameters: string[] = [];
if (scope.host) { conditions.push('o.host_id=?'); parameters.push(scope.host); }
if (scope.server) { conditions.push('o.server_name=?'); parameters.push(scope.server); }
if (scope.operation) { conditions.push('o.id=?'); parameters.push(scope.operation); }
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
return handle.transaction((): HostOperationHistoryPage => {
const countColumn = filter === 'all' ? 'total' : filter;
const total = (handle.prepare(`SELECT ${countColumn} AS total FROM ec_host_operation_history_counts WHERE host_id=? AND server_name=?`)
.get(scope.host ?? '', scope.server ?? '') as { total: number } | undefined)?.total ?? 0;
// An exact operation seeks its unique ID instead of paging through history.
const total = scope.operation
? (handle.prepare(`SELECT 1 AS total FROM ec_host_operations o ${where}`).get(...parameters) as { total: number } | undefined)?.total ?? 0
: (handle.prepare(`SELECT ${countColumn} AS total FROM ec_host_operation_history_counts WHERE host_id=? AND server_name=?`)
.get(scope.host ?? '', scope.server ?? '') as { total: number } | undefined)?.total ?? 0;
const totalPages = Math.max(1, Math.ceil(total / pageSize));
const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1);
const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind,
src/lib/server/maintenance/index.ts
+2 -2
View File
@@ -3,8 +3,8 @@ import { watchdogOverview, watchdogTick } from './recovery.js';
import { maintenanceTick, plutoniumStatus } from './runtime.js';
import type { MaintenanceOverview } from '../../maintenance.js';
export function maintenanceOverview(): MaintenanceOverview {
return { policies: maintenancePolicies(), recovery: watchdogOverview(), runtime: plutoniumStatus() };
export function maintenanceOverview(selectedJob: number | null = null): MaintenanceOverview {
return { policies: maintenancePolicies(), recovery: watchdogOverview(), runtime: plutoniumStatus(selectedJob) };
}
export async function runMaintenance(): Promise<void> {
// One failing observer must not prevent the other capabilities from reconciling.
src/lib/server/maintenance/runtime.ts
+2 -2
View File
@@ -18,14 +18,14 @@ export function saveMaintenanceSchedule(enabled: boolean, time: string, timeZone
const nextAt = nextMaintenanceAt(time, new Date(), timeZone);
writeSetting('plutonium_schedule', { enabled, time, timeZone, nextAt: enabled ? nextAt : null });
}
export function plutoniumStatus(): PlutoniumStatus {
export function plutoniumStatus(selectedJob: number | null = null): PlutoniumStatus {
return {
release: release(), schedule: schedule(), servers: runningVersions(),
jobs: db().prepare(`SELECT m.*, d.stage AS operation_stage, d.state AS operation_state
FROM maintenance_jobs m LEFT JOIN server_deployments d ON d.id = m.provisioning_id
WHERE m.state IN ('queued','restarting')
OR (m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id))
OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) ORDER BY m.id DESC`).all() as MaintenanceJob[]
OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) OR m.id = ? ORDER BY m.id DESC`).all(selectedJob) as MaintenanceJob[]
};
}