Unify retained operation history in Activity
This commit is contained in:
@@ -0,0 +1,188 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
import { normalizeActivityQuery, type ActivityPage, type ActivityQuery, type ActivityRow, type ActivitySource } from '../activity.js';
|
||||
import { serverTitle } from '../ui/server-labels.js';
|
||||
import { db } from './db/index.js';
|
||||
import { ensureHostSchema } from './hosts/schema.js';
|
||||
import { importantOperationSql } from './hosts/observations.js';
|
||||
|
||||
const initialized = new WeakSet<Database.Database>();
|
||||
const quote = (value: string) => `'${value.replaceAll("'", "''")}'`;
|
||||
const hostTitles: Record<string, string> = {
|
||||
'instance.create': 'Create server container', 'instance.rebuild': 'Rebuild server container',
|
||||
'instance.start': 'Start server container', 'instance.restart': 'Restart server container',
|
||||
'instance.stop': 'Stop server container', 'instance.remove': 'Remove server container',
|
||||
'instance.cancel-create': 'Cancel server creation', 'instance.inspect': 'Check server container',
|
||||
'host.inspect': 'Check host', 'host.decommission': 'Remove host', 'runtime.pull': 'Prepare container runtime',
|
||||
'library.download': 'Download game files', 'library.import': 'Import game files',
|
||||
'plutonium.prepare': 'Prepare game update', 'files.stage': 'Stage server files',
|
||||
'files.apply': 'Apply server files', 'files.read': 'Read server files', 'files.list': 'List server files',
|
||||
'files.stat': 'Check server files', 'files.manifest': 'Check file versions', 'files.remove': 'Remove server files',
|
||||
'rcon.status-batch': 'Check game status', 'rcon.execute': 'Game command', 'logs.read': 'Read server logs'
|
||||
};
|
||||
const labelCase = (column: string, labels: Record<string, string>, fallback: string) =>
|
||||
`CASE ${column} ${Object.entries(labels).map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`).join(' ')} ELSE ${quote(fallback)} END`;
|
||||
|
||||
// Old attempts may no longer be the native_provisioning pointer. Their stable
|
||||
// idempotency keys still identify the same retained controller operation.
|
||||
const lifecycleKey = `CASE
|
||||
WHEN substr(o.idempotency_key,-6)=':stage' THEN substr(o.idempotency_key,1,length(o.idempotency_key)-6)
|
||||
${[':migration-create:', ':migrate-remove:'].map(marker => `WHEN instr(o.idempotency_key,${quote(marker)})>0
|
||||
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length})<>''
|
||||
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length}) NOT GLOB '*[^0-9]*'
|
||||
THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`).join('\n')}
|
||||
ELSE o.idempotency_key END`;
|
||||
|
||||
/** One row per user-level operation or finding; no command payloads or observations are read. */
|
||||
const activitySql = `WITH
|
||||
links AS (
|
||||
SELECT operation_id,kind,source_id,server FROM provisioning_sources
|
||||
UNION SELECT id,substr(triggered_by,1,instr(triggered_by,':')-1),substr(triggered_by,instr(triggered_by,':')+1),server
|
||||
FROM server_deployments WHERE substr(triggered_by,1,instr(triggered_by,':')-1) IN ('maintenance','watchdog','campaign','beta','autoscaling')
|
||||
UNION SELECT provisioning_id,'maintenance',CAST(id AS TEXT),server FROM maintenance_jobs WHERE provisioning_id IS NOT NULL
|
||||
UNION SELECT provisioning_id,'campaign',deployment_id,server FROM deployment_servers WHERE provisioning_id IS NOT NULL
|
||||
UNION SELECT provisioning_id,'beta',CAST(beta_id AS TEXT),server FROM beta_servers WHERE provisioning_id IS NOT NULL
|
||||
UNION SELECT a.operation_id,'watchdog',i.id,i.server FROM watchdog_actions a JOIN watchdog_incidents i ON i.id=a.incident_id
|
||||
),
|
||||
memberships AS (
|
||||
SELECT operation_id,'|' || replace(group_concat(DISTINCT CASE kind WHEN 'watchdog' THEN 'recovery' ELSE kind END),',','|') || '|' AS sources
|
||||
FROM links GROUP BY operation_id
|
||||
),
|
||||
maintenance_links AS (
|
||||
SELECT DISTINCT l.operation_id,m.id,m.state,m.kind,m.target_revision
|
||||
FROM links l JOIN maintenance_jobs m ON l.kind='maintenance' AND l.source_id=CAST(m.id AS TEXT) AND l.server=m.server
|
||||
JOIN server_deployments d ON d.id=l.operation_id
|
||||
WHERE m.provisioning_id=l.operation_id OR (m.provisioning_id IS NULL AND NOT EXISTS (
|
||||
SELECT 1 FROM links newer JOIN server_deployments n ON n.id=newer.operation_id
|
||||
WHERE newer.kind='maintenance' AND newer.source_id=l.source_id AND newer.server=l.server
|
||||
AND (n.started_at>d.started_at OR (n.started_at=d.started_at AND n.rowid>d.rowid))))
|
||||
),
|
||||
maintenance_results AS (
|
||||
SELECT operation_id,MAX(state='failed') AS failed,MAX(state IN ('queued','restarting')) AS pending,
|
||||
MAX(state='cancelled') AS cancelled,MAX(kind='update') AS is_update,MAX(target_revision) AS revision
|
||||
FROM maintenance_links GROUP BY operation_id
|
||||
),
|
||||
covered_host AS (
|
||||
SELECT n.operation_id AS id FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.operation_id IS NOT NULL
|
||||
UNION SELECT n.stage_operation_id FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.stage_operation_id IS NOT NULL
|
||||
UNION SELECT m.remove_operation_id FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.remove_operation_id IS NOT NULL
|
||||
),
|
||||
items AS (
|
||||
SELECT 'provisioning:' || d.id AS id,d.started_at AS at,
|
||||
CASE WHEN m.is_update=1 THEN 'update' WHEN d.action='destroy' THEN 'remove' ELSE d.action END AS type,
|
||||
COALESCE(ms.sources,'|provisioning|') AS sources,
|
||||
CASE WHEN d.state IN ('failed','interrupted','cancelled','warning') THEN d.state
|
||||
WHEN m.failed=1 THEN 'failed' WHEN m.pending=1 AND d.state='done' THEN 'verifying'
|
||||
WHEN m.cancelled=1 THEN 'cancelled' ELSE d.state END AS rawState,
|
||||
CASE WHEN m.is_update=1 THEN 'Update game' ELSE ${labelCase('d.action', { create: 'Create server', start: 'Start server', restart: 'Restart server', rebuild: 'Rebuild server', stop: 'Stop server', destroy: 'Remove server' }, 'Server operation')} END AS title,
|
||||
CASE WHEN m.failed=1 AND d.state='done' THEN 'The restart completed, but maintenance verification failed.'
|
||||
WHEN m.pending=1 AND d.state='done' THEN 'The restart completed. Maintenance verification is still in progress.'
|
||||
WHEN m.revision IS NOT NULL THEN 'Target game revision r' || m.revision || '.'
|
||||
WHEN d.state='waiting' THEN 'Waiting for eligibility and available capacity.'
|
||||
WHEN d.state='awaiting-health' THEN 'Checking that the game is responding.' ELSE NULL END AS detail,
|
||||
d.server,d.server_deleted_at AS serverDeletedAt,COALESCE(n.host_id,mg.source_host_id) AS hostId,mg.source_host_id AS secondaryHostId,
|
||||
'provisioning' AS route,d.id AS recordId
|
||||
FROM server_deployments d LEFT JOIN memberships ms ON ms.operation_id=d.id
|
||||
LEFT JOIN maintenance_results m ON m.operation_id=d.id
|
||||
LEFT JOIN native_provisioning p ON p.deployment_id=d.id LEFT JOIN native_instances n ON n.id=p.instance_id
|
||||
LEFT JOIN ec_host_migrations mg ON mg.deployment_id=d.id
|
||||
UNION ALL
|
||||
SELECT 'maintenance:' || m.id,m.requested_at,CASE WHEN m.kind='update' THEN 'update' ELSE 'restart' END,
|
||||
'|maintenance|',m.state,CASE WHEN m.kind='update' THEN 'Update game' WHEN m.kind='schedule' THEN 'Scheduled server maintenance' ELSE 'Server maintenance' END,
|
||||
CASE WHEN m.target_revision IS NOT NULL THEN 'Target game revision r' || m.target_revision || '.' ELSE NULL END,
|
||||
m.server,m.server_deleted_at,NULL,NULL,'maintenance-job',CAST(m.id AS TEXT)
|
||||
FROM maintenance_jobs m WHERE NOT EXISTS (SELECT 1 FROM maintenance_links l WHERE l.id=m.id)
|
||||
UNION ALL
|
||||
SELECT 'recovery:' || i.id,i.first_seen_at,i.kind,'|recovery|',i.state,
|
||||
CASE WHEN i.kind='health' THEN 'Game health finding' ELSE 'Server files differ from the promoted setup' END,
|
||||
CASE WHEN i.state='recovering' THEN 'Recovery is in progress.' WHEN i.state='attention' THEN 'Automatic recovery needs review.'
|
||||
WHEN i.state='resolved' THEN 'The finding was resolved.' WHEN i.state='dismissed' THEN 'The finding was dismissed.' ELSE 'A check found an issue that needs review.' END,
|
||||
i.server,i.server_deleted_at,NULL,NULL,'recovery',i.id
|
||||
FROM watchdog_incidents i
|
||||
UNION ALL
|
||||
SELECT 'campaign:' || d.id || ':' || ds.server,d.started_at,CASE WHEN d.action='drift' THEN 'drift' ELSE 'files' END,
|
||||
CASE WHEN d.triggered_by='watchdog' OR EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=d.id)
|
||||
THEN '|campaign|recovery|' ELSE '|campaign|' END,ds.restart_state,
|
||||
CASE d.action WHEN 'promote' THEN 'Promote files' WHEN 'rollback' THEN 'Roll back files' ELSE 'Reconcile files' END,
|
||||
CAST(ds.files_changed AS TEXT) || CASE WHEN ds.files_changed=1 THEN ' file changed.' ELSE ' files changed.' END,
|
||||
ds.server,ds.server_deleted_at,NULL,NULL,'campaign',d.id
|
||||
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id=d.id
|
||||
UNION ALL
|
||||
SELECT 'beta:' || b.id || ':' || bs.server,b.opened_at,'beta','|beta|',
|
||||
CASE WHEN b.state IN ('promoted','rolled-back','cancelled') THEN b.state ELSE bs.stage END,
|
||||
'Beta test #' || b.id,NULL,bs.server,bs.server_deleted_at,NULL,NULL,'beta',CAST(b.id AS TEXT)
|
||||
FROM betas b JOIN beta_servers bs ON bs.beta_id=b.id
|
||||
UNION ALL
|
||||
SELECT 'host:' || o.id,o.created_at,'host','|host|',
|
||||
CASE WHEN o.state='completed' AND o.routine=1 AND o.observation_success=0 THEN 'observation-failed'
|
||||
WHEN o.state='completed' AND COALESCE(o.error,'')<>'' THEN 'warning' ELSE o.state END,
|
||||
${labelCase('o.kind', hostTitles, 'Host operation')},
|
||||
CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0 THEN 'The command completed, but the check reported a problem.'
|
||||
WHEN o.state='unknown' THEN 'The host has not confirmed the result.' ELSE NULL END,
|
||||
COALESCE(o.server_name,n.server),s.destroyed_at,o.host_id,NULL,'host',o.id
|
||||
FROM ec_host_operations o LEFT JOIN native_instances n ON n.id=o.instance_id
|
||||
LEFT JOIN servers s ON s.name=COALESCE(o.server_name,n.server)
|
||||
WHERE o.history_expired_at IS NULL AND ${importantOperationSql}
|
||||
AND NOT EXISTS (SELECT 1 FROM covered_host c WHERE c.id=o.id)
|
||||
AND NOT (o.kind IN ('files.stage','instance.create','instance.rebuild','instance.start','instance.restart','instance.stop','instance.remove','instance.cancel-create')
|
||||
AND EXISTS (SELECT 1 FROM server_deployments d WHERE d.id=${lifecycleKey}))
|
||||
),
|
||||
normalized AS (
|
||||
SELECT i.*,h.name AS hostName,
|
||||
CASE WHEN rawState IN ('queued','pending','waiting','held','awaiting-empty','awaiting-match') THEN 'queued'
|
||||
WHEN rawState IN ('running','accepted','preparing','staging','testing','observing','deploying','restarting','recovering','awaiting-health','verifying','rolling-back') THEN 'running'
|
||||
WHEN rawState IN ('done','completed','resolved','ready','ready-promote','promoted','rolled-back') THEN 'succeeded'
|
||||
WHEN rawState IN ('failed','interrupted','observation-failed') THEN 'failed'
|
||||
WHEN rawState IN ('cancelled','dismissed','skipped') THEN 'cancelled'
|
||||
WHEN rawState IN ('manual','attention','open') THEN 'attention'
|
||||
WHEN rawState='warning' THEN 'warning' ELSE 'unknown' END AS state
|
||||
FROM items i LEFT JOIN ec_hosts h ON h.id=i.hostId
|
||||
)
|
||||
`;
|
||||
|
||||
type SqlRow = Omit<ActivityRow, 'source' | 'sources' | 'href'> & { sources: string; route: string; recordId: string; secondaryHostId: string | null };
|
||||
const sourceOrder: ActivitySource[] = ['maintenance', 'recovery', 'campaign', 'beta', 'autoscaling', 'provisioning', 'host'];
|
||||
function present(row: SqlRow): ActivityRow {
|
||||
const sources = sourceOrder.filter(source => row.sources.includes(`|${source}|`));
|
||||
const href = row.route === 'maintenance-job' ? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}`
|
||||
: row.route === 'recovery' ? `/maintenance/${encodeURIComponent(row.recordId)}`
|
||||
: row.route === 'host' ? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all`
|
||||
: `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`;
|
||||
return { id: row.id, at: row.at, type: row.type, source: sources[0] ?? 'provisioning', sources,
|
||||
state: row.state, rawState: row.rawState, title: row.title, detail: row.detail,
|
||||
server: row.server, serverDeletedAt: row.serverDeletedAt, hostId: row.hostId, hostName: row.hostName, href };
|
||||
}
|
||||
|
||||
/** All predicates, totals and pagination operate on the same retained SQL union. */
|
||||
export function activityHistory(input: ActivityQuery = {}, now = Date.now()): ActivityPage {
|
||||
const query = normalizeActivityQuery(input);
|
||||
const handle = db();
|
||||
ensureHostSchema(handle);
|
||||
if (!initialized.has(handle)) {
|
||||
handle.function('activity_server_title', { deterministic: true }, (name: unknown) => typeof name === 'string' ? serverTitle(name) : '');
|
||||
initialized.add(handle);
|
||||
}
|
||||
const conditions: string[] = [];
|
||||
const bindings: Record<string, string | number> = {};
|
||||
for (const key of ['server', 'type', 'state'] as const) if (query[key]) { conditions.push(`${key}=@${key}`); bindings[key] = query[key]; }
|
||||
if (query.host) { conditions.push('(hostId=@host OR secondaryHostId=@host)'); bindings.host = query.host; }
|
||||
if (query.source) {
|
||||
// Maintenance is the user-facing umbrella; Recovery remains available as a narrower filter.
|
||||
conditions.push(query.source === 'maintenance' ? "(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)" : 'instr(sources,@source)>0');
|
||||
if (query.source !== 'maintenance') bindings.source = `|${query.source}|`;
|
||||
}
|
||||
if (query.days) { conditions.push('julianday(at)>=julianday(@since)'); bindings.since = new Date(now - query.days * 86_400_000).toISOString(); }
|
||||
if (query.q) {
|
||||
conditions.push("instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0");
|
||||
bindings.q = query.q.toLocaleLowerCase();
|
||||
}
|
||||
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
const limit = query.limit!;
|
||||
return handle.transaction(() => {
|
||||
const { total } = handle.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`).get(bindings) as { total: number };
|
||||
const offset = Math.min(query.offset!, Math.max(0, Math.ceil(total / limit) - 1) * limit);
|
||||
const rows = handle.prepare(`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId
|
||||
FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`)
|
||||
.all({ ...bindings, limit, offset }) as SqlRow[];
|
||||
return { rows: rows.map(present), total, limit, offset, query: { ...query, offset } };
|
||||
})();
|
||||
}
|
||||
Reference in New Issue
Block a user