import { hostOperationTitles } from '../host-operation-labels.js'; import type Database from 'better-sqlite3'; import { normalizeActivityQuery, type ActivityPage, type ActivityQuery, type ActivityRow, type ActivitySource } from '../activity.js'; import { serverTitle } from '../ui/server-labels.js'; import { db } from './db/index.js'; import { ensureHostSchema } from './hosts/schema.js'; import { importantOperationSql } from './hosts/observations.js'; const initialized = new WeakSet(); const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; const labelCase = (column: string, labels: Record, fallback: string) => `CASE ${column} ${Object.entries(labels).map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`).join(' ')} ELSE ${quote(fallback)} END`; // Old attempts may no longer be the native_provisioning pointer. Their stable // idempotency keys still identify the same retained controller operation. const lifecycleKey = `CASE WHEN substr(o.idempotency_key,-6)=':stage' THEN substr(o.idempotency_key,1,length(o.idempotency_key)-6) ${[':migration-create:', ':migrate-remove:'].map(marker => `WHEN instr(o.idempotency_key,${quote(marker)})>0 AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length})<>'' AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length}) NOT GLOB '*[^0-9]*' THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`).join('\n')} ELSE o.idempotency_key END`; /** One row per user-level operation or finding; no command payloads or observations are read. */ const activitySql = `WITH links AS ( SELECT operation_id,kind,source_id,server FROM provisioning_sources UNION SELECT id,substr(triggered_by,1,instr(triggered_by,':')-1),substr(triggered_by,instr(triggered_by,':')+1),server FROM server_deployments WHERE substr(triggered_by,1,instr(triggered_by,':')-1) IN ('maintenance','watchdog','campaign','beta','autoscaling') UNION SELECT provisioning_id,'maintenance',CAST(id AS TEXT),server FROM maintenance_jobs WHERE provisioning_id IS NOT NULL UNION SELECT provisioning_id,'campaign',deployment_id,server FROM deployment_servers WHERE provisioning_id IS NOT NULL UNION SELECT provisioning_id,'beta',CAST(beta_id AS TEXT),server FROM beta_servers WHERE provisioning_id IS NOT NULL UNION SELECT a.operation_id,'watchdog',i.id,i.server FROM watchdog_actions a JOIN watchdog_incidents i ON i.id=a.incident_id ), memberships AS ( SELECT operation_id,'|' || replace(group_concat(DISTINCT CASE kind WHEN 'watchdog' THEN 'recovery' ELSE kind END),',','|') || '|' AS sources FROM links GROUP BY operation_id ), maintenance_links AS ( SELECT DISTINCT l.operation_id,m.id,m.state,m.kind,m.target_revision FROM links l JOIN maintenance_jobs m ON l.kind='maintenance' AND l.source_id=CAST(m.id AS TEXT) AND l.server=m.server JOIN server_deployments d ON d.id=l.operation_id WHERE m.provisioning_id=l.operation_id OR (m.provisioning_id IS NULL AND NOT EXISTS ( SELECT 1 FROM links newer JOIN server_deployments n ON n.id=newer.operation_id WHERE newer.kind='maintenance' AND newer.source_id=l.source_id AND newer.server=l.server AND (n.started_at>d.started_at OR (n.started_at=d.started_at AND n.rowid>d.rowid)))) ), maintenance_results AS ( SELECT operation_id,MAX(state='failed') AS failed,MAX(state IN ('queued','restarting')) AS pending, MAX(state='cancelled') AS cancelled,MAX(kind='update') AS is_update,MAX(target_revision) AS revision FROM maintenance_links GROUP BY operation_id ), covered_host AS ( SELECT n.operation_id AS id FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.operation_id IS NOT NULL UNION SELECT n.stage_operation_id FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.stage_operation_id IS NOT NULL UNION SELECT m.remove_operation_id FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.remove_operation_id IS NOT NULL ), items AS ( SELECT 'provisioning:' || d.id AS id,d.started_at AS at, CASE WHEN m.is_update=1 THEN 'update' WHEN d.action='destroy' THEN 'remove' ELSE d.action END AS type, COALESCE(ms.sources,'|provisioning|') AS sources, CASE WHEN d.state IN ('failed','interrupted','cancelled','warning') THEN d.state WHEN m.failed=1 THEN 'failed' WHEN m.pending=1 AND d.state='done' THEN 'verifying' WHEN m.cancelled=1 THEN 'cancelled' ELSE d.state END AS rawState, CASE WHEN m.is_update=1 THEN 'Update game' ELSE ${labelCase('d.action', { create: 'Create server', start: 'Start server', restart: 'Restart server', rebuild: 'Rebuild server', stop: 'Stop server', destroy: 'Remove server' }, 'Server operation')} END AS title, CASE WHEN m.failed=1 AND d.state='done' THEN 'The restart completed, but maintenance verification failed.' WHEN m.pending=1 AND d.state='done' THEN 'The restart completed. Maintenance verification is still in progress.' WHEN m.revision IS NOT NULL THEN 'Target game revision r' || m.revision || '.' WHEN d.state='waiting' THEN 'Waiting for eligibility and available capacity.' WHEN d.state='awaiting-health' THEN 'Checking that the game is responding.' ELSE NULL END AS detail, d.server,d.server_deleted_at AS serverDeletedAt,COALESCE(n.host_id,mg.source_host_id) AS hostId,mg.source_host_id AS secondaryHostId, 'provisioning' AS route,d.id AS recordId FROM server_deployments d LEFT JOIN memberships ms ON ms.operation_id=d.id LEFT JOIN maintenance_results m ON m.operation_id=d.id LEFT JOIN native_provisioning p ON p.deployment_id=d.id LEFT JOIN native_instances n ON n.id=p.instance_id LEFT JOIN ec_host_migrations mg ON mg.deployment_id=d.id UNION ALL SELECT 'maintenance:' || m.id,m.requested_at,CASE WHEN m.kind='update' THEN 'update' ELSE 'restart' END, '|maintenance|',m.state,CASE WHEN m.kind='update' THEN 'Update game' WHEN m.kind='schedule' THEN 'Scheduled server maintenance' ELSE 'Server maintenance' END, CASE WHEN m.target_revision IS NOT NULL THEN 'Target game revision r' || m.target_revision || '.' ELSE NULL END, m.server,m.server_deleted_at,NULL,NULL,'maintenance-job',CAST(m.id AS TEXT) FROM maintenance_jobs m WHERE NOT EXISTS (SELECT 1 FROM maintenance_links l WHERE l.id=m.id) UNION ALL SELECT 'recovery:' || i.id,i.first_seen_at,i.kind,'|recovery|',i.state, CASE WHEN i.kind='health' THEN 'Game health finding' ELSE 'Server files differ from the promoted setup' END, CASE WHEN i.state='recovering' THEN 'Recovery is in progress.' WHEN i.state='attention' THEN 'Automatic recovery needs review.' WHEN i.state='resolved' THEN 'The finding was resolved.' WHEN i.state='dismissed' THEN 'The finding was dismissed.' ELSE 'A check found an issue that needs review.' END, i.server,i.server_deleted_at,NULL,NULL,'recovery',i.id FROM watchdog_incidents i UNION ALL SELECT 'campaign:' || d.id || ':' || ds.server,d.started_at,CASE WHEN d.action='drift' THEN 'drift' ELSE 'files' END, CASE WHEN d.triggered_by='watchdog' OR EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=d.id) THEN '|campaign|recovery|' ELSE '|campaign|' END,ds.restart_state, CASE d.action WHEN 'promote' THEN 'Promote files' WHEN 'rollback' THEN 'Roll back files' ELSE 'Reconcile files' END, CAST(ds.files_changed AS TEXT) || CASE WHEN ds.files_changed=1 THEN ' file changed.' ELSE ' files changed.' END, ds.server,ds.server_deleted_at,NULL,NULL,'campaign',d.id FROM deployments d JOIN deployment_servers ds ON ds.deployment_id=d.id UNION ALL SELECT 'beta:' || b.id || ':' || bs.server,b.opened_at,'beta','|beta|', CASE WHEN b.state IN ('promoted','rolled-back','cancelled') THEN b.state ELSE bs.stage END, 'Beta test #' || b.id,NULL,bs.server,bs.server_deleted_at,NULL,NULL,'beta',CAST(b.id AS TEXT) FROM betas b JOIN beta_servers bs ON bs.beta_id=b.id UNION ALL SELECT 'host:' || o.id,o.created_at,'host','|host|', CASE WHEN o.state='completed' AND o.routine=1 AND o.observation_success=0 THEN 'observation-failed' WHEN o.state='completed' AND COALESCE(o.error,'')<>'' THEN 'warning' ELSE o.state END, ${labelCase('o.kind', hostOperationTitles, 'Host operation')}, CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0 THEN 'The command completed, but the check reported a problem.' WHEN o.state='unknown' THEN 'The host has not confirmed the result.' ELSE NULL END, COALESCE(o.server_name,n.server),s.destroyed_at,o.host_id,NULL,'host',o.id FROM ec_host_operations o LEFT JOIN native_instances n ON n.id=o.instance_id LEFT JOIN servers s ON s.name=COALESCE(o.server_name,n.server) WHERE o.history_expired_at IS NULL AND ${importantOperationSql} AND NOT EXISTS (SELECT 1 FROM covered_host c WHERE c.id=o.id) AND NOT (o.kind IN ('files.stage','instance.create','instance.rebuild','instance.start','instance.restart','instance.stop','instance.remove','instance.cancel-create') AND EXISTS (SELECT 1 FROM server_deployments d WHERE d.id=${lifecycleKey})) ), normalized AS ( SELECT i.*,h.name AS hostName, CASE WHEN rawState IN ('queued','pending','waiting','held','awaiting-empty','awaiting-match') THEN 'queued' WHEN rawState IN ('running','accepted','preparing','staging','testing','observing','deploying','restarting','recovering','awaiting-health','verifying','rolling-back') THEN 'running' WHEN rawState IN ('done','completed','resolved','ready','ready-promote','promoted','rolled-back') THEN 'succeeded' WHEN rawState IN ('failed','interrupted','observation-failed') THEN 'failed' WHEN rawState IN ('cancelled','dismissed','skipped') THEN 'cancelled' WHEN rawState IN ('manual','attention','open') THEN 'attention' WHEN rawState='warning' THEN 'warning' ELSE 'unknown' END AS state FROM items i LEFT JOIN ec_hosts h ON h.id=i.hostId ) `; type SqlRow = Omit & { sources: string; route: string; recordId: string; secondaryHostId: string | null }; const sourceOrder: ActivitySource[] = ['maintenance', 'recovery', 'campaign', 'beta', 'autoscaling', 'provisioning', 'host']; function present(row: SqlRow): ActivityRow { const sources = sourceOrder.filter(source => row.sources.includes(`|${source}|`)); const href = row.route === 'maintenance-job' ? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}` : row.route === 'recovery' ? `/maintenance/${encodeURIComponent(row.recordId)}` : row.route === 'host' ? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all` : `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`; return { id: row.id, at: row.at, type: row.type, source: sources[0] ?? 'provisioning', sources, state: row.state, rawState: row.rawState, title: row.title, detail: row.detail, server: row.server, serverDeletedAt: row.serverDeletedAt, hostId: row.hostId, hostName: row.hostName, href }; } /** All predicates, totals and pagination operate on the same retained SQL union. */ export function activityHistory(input: ActivityQuery = {}, now = Date.now()): ActivityPage { const query = normalizeActivityQuery(input); const handle = db(); ensureHostSchema(handle); if (!initialized.has(handle)) { handle.function('activity_server_title', { deterministic: true }, (name: unknown) => typeof name === 'string' ? serverTitle(name) : ''); initialized.add(handle); } const conditions: string[] = []; const bindings: Record = {}; for (const key of ['server', 'type', 'state'] as const) if (query[key]) { conditions.push(`${key}=@${key}`); bindings[key] = query[key]; } if (query.host) { conditions.push('(hostId=@host OR secondaryHostId=@host)'); bindings.host = query.host; } if (query.source) { // Maintenance is the user-facing umbrella; Recovery remains available as a narrower filter. conditions.push(query.source === 'maintenance' ? "(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)" : 'instr(sources,@source)>0'); if (query.source !== 'maintenance') bindings.source = `|${query.source}|`; } if (query.days) { conditions.push('julianday(at)>=julianday(@since)'); bindings.since = new Date(now - query.days * 86_400_000).toISOString(); } if (query.q) { conditions.push("instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0"); bindings.q = query.q.toLocaleLowerCase(); } const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : ''; const limit = query.limit!; return handle.transaction(() => { const { total } = handle.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`).get(bindings) as { total: number }; const offset = Math.min(query.offset!, Math.max(0, Math.ceil(total / limit) - 1) * limit); const rows = handle.prepare(`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`) .all({ ...bindings, limit, offset }) as SqlRow[]; return { rows: rows.map(present), total, limit, offset, query: { ...query, offset } }; })(); }