Keep artifact list and beta queries free of file bytes

This commit is contained in:
2026-10-06 14:01:12 -04:00
parent 27be1abf41
commit ddb3db9537
21 changed files with 197 additions and 101 deletions
+2
View File
@@ -11,6 +11,7 @@ import { initializeFleetSettings } from './fleet-settings.js';
import { migrateServerHistory } from './server-history.js'; import { migrateServerHistory } from './server-history.js';
import { migrateNativeInstances } from '../backends/schema.js'; import { migrateNativeInstances } from '../backends/schema.js';
import { installPerformanceIndexes } from './performance-indexes.js'; import { installPerformanceIndexes } from './performance-indexes.js';
import { migrateVersionLayout } from './version-layout.js';
import { timeStatement } from './slow-statements.js'; import { timeStatement } from './slow-statements.js';
let _db: Database.Database | null = null; let _db: Database.Database | null = null;
@@ -388,6 +389,7 @@ export function db(): Database.Database {
backfillLeaderboard(handle); backfillLeaderboard(handle);
initializeFleetSettings(handle); initializeFleetSettings(handle);
migrateNativeInstances(handle); migrateNativeInstances(handle);
migrateVersionLayout(handle);
// Lifecycle history is retained indefinitely. Admission and status reads // Lifecycle history is retained indefinitely. Admission and status reads
// must narrow by server or active state instead of scanning that history. // must narrow by server or active state instead of scanning that history.
// Install after migrations that rebuild this table or add server_deleted_at. // Install after migrations that rebuild this table or add server_deleted_at.
+1 -1
View File
@@ -130,11 +130,11 @@ CREATE TABLE IF NOT EXISTS script_versions (
release_version TEXT, -- publisher's version/tag, independent of upload order release_version TEXT, -- publisher's version/tag, independent of upload order
sha256 TEXT NOT NULL, sha256 TEXT NOT NULL,
size INTEGER NOT NULL, size INTEGER NOT NULL,
content BLOB NOT NULL,
notes TEXT, notes TEXT,
state TEXT NOT NULL DEFAULT 'untested', -- 'untested' | 'tested' | 'test-failed' state TEXT NOT NULL DEFAULT 'untested', -- 'untested' | 'tested' | 'test-failed'
kind TEXT NOT NULL DEFAULT 'script', -- denormalized from scripts.kind for join-free filtering kind TEXT NOT NULL DEFAULT 'script', -- denormalized from scripts.kind for join-free filtering
uploaded_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), uploaded_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
content BLOB NOT NULL,
UNIQUE(script_id, version) UNIQUE(script_id, version)
); );
CREATE INDEX IF NOT EXISTS idx_script_versions_script ON script_versions(script_id); CREATE INDEX IF NOT EXISTS idx_script_versions_script ON script_versions(script_id);
+27
View File
@@ -0,0 +1,27 @@
import type Database from 'better-sqlite3';
/** Keep metadata ahead of overflow pages, preserving IDs and all references. */
export function migrateVersionLayout(handle: Database.Database): void {
const columns = handle.pragma('table_info(script_versions)') as { name: string }[];
if (columns.at(-1)?.name === 'content') return;
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'").get() as { sql: string };
const content = /\bcontent\s+BLOB\s+NOT\s+NULL\s*,/i;
if (!content.test(sql)) throw new Error('Unrecognized script version content layout');
const replacement = sql.replace(content, '').replace(/,\s*UNIQUE\s*\(/i, ', content BLOB NOT NULL, UNIQUE(')
.replace('script_versions', 'script_versions_metadata');
const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='script_versions' AND type IN ('index','trigger') AND sql IS NOT NULL")
.all() as { sql: string }[];
const sequence = handle.prepare("SELECT seq FROM sqlite_sequence WHERE name='script_versions'").get() as { seq: number } | undefined;
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement);
const names = columns.map(column => `"${column.name}"`).join(',');
handle.exec(`INSERT INTO script_versions_metadata (${names}) SELECT ${names} FROM script_versions;
DROP TABLE script_versions; ALTER TABLE script_versions_metadata RENAME TO script_versions;`);
if (sequence) handle.prepare("UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name='script_versions'").run(sequence.seq);
for (const dependent of dependents) handle.exec(dependent.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Version layout migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
}
+3 -2
View File
@@ -21,9 +21,10 @@ export function archiveBetaRevision(id: number, revision: number, startedAt: str
export function betaRevisionHistory(id: number): BetaRevision[] { export function betaRevisionHistory(id: number): BetaRevision[] {
const rows = db().prepare('SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC') const rows = db().prepare('SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC')
.all(id) as Array<{ revision: number; started_at: string; notes: string | null; results_json: string | null }>; .all(id) as Array<{ revision: number; started_at: string; notes: string | null; results_json: string | null }>;
const files = db().prepare(`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`).all(id) as Array<BetaRevision['files'][number] & { revision: number }>;
return rows.map(({ results_json, ...row }) => ({ ...row, return rows.map(({ results_json, ...row }) => ({ ...row,
results: results_json ? JSON.parse(results_json) : null, results: results_json ? JSON.parse(results_json) : null,
files: db().prepare(`SELECT script_id, name, kind, path, version_id, version, release_version, sha256 files: files.filter(file => file.revision === row.revision).map(({ revision, ...file }) => file)
FROM beta_revision_files WHERE beta_id = ? AND revision = ? ORDER BY script_id`).all(id, row.revision) as BetaRevision['files']
})); }));
} }
+11 -10
View File
@@ -1,3 +1,4 @@
import { getVersionContent } from './scripts.js';
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
import { storedPlan, testView, activity, setOutcome } from './beta-tests.js'; import { storedPlan, testView, activity, setOutcome } from './beta-tests.js';
import { archiveBetaRevision, currentBetaRevision } from './beta-revisions.js'; import { archiveBetaRevision, currentBetaRevision } from './beta-revisions.js';
@@ -89,7 +90,7 @@ function composeCfgForServer(
if (inputs.inBetaBaseline) { if (inputs.inBetaBaseline) {
const s = getScript(inputs.inBetaBaseline.script_id); const s = getScript(inputs.inBetaBaseline.script_id);
if (s && scopeAppliesToServer(s, { game: serverGame, map: serverMap })) { if (s && scopeAppliesToServer(s, { game: serverGame, map: serverMap })) {
baselineText = inputs.inBetaBaseline.content.toString('utf8'); baselineText = getVersionContent(inputs.inBetaBaseline.id).toString('utf8');
baselineScriptId = s.id; baselineScriptId = s.id;
} }
} }
@@ -99,7 +100,7 @@ function composeCfgForServer(
if (promoted?.current_version_id) { if (promoted?.current_version_id) {
const ver = getVersion(promoted.current_version_id); const ver = getVersion(promoted.current_version_id);
if (ver) { if (ver) {
baselineText = ver.content.toString('utf8'); baselineText = getVersionContent(ver.id).toString('utf8');
baselineScriptId = promoted.id; baselineScriptId = promoted.id;
} }
} }
@@ -122,7 +123,7 @@ function composeCfgForServer(
if (!s.current_version_id) continue; if (!s.current_version_id) continue;
const ver = getVersion(s.current_version_id); const ver = getVersion(s.current_version_id);
if (!ver) continue; if (!ver) continue;
promotedOverlays.push({ id: s.id, text: ver.content.toString('utf8') }); promotedOverlays.push({ id: s.id, text: getVersionContent(ver.id).toString('utf8') });
promotedIdsSeen.add(s.id); promotedIdsSeen.add(s.id);
} }
}; };
@@ -132,7 +133,7 @@ function composeCfgForServer(
// In-beta overlays scoped to this server. // In-beta overlays scoped to this server.
const inBetaScopedOverlays = inputs.inBetaOverlays const inBetaScopedOverlays = inputs.inBetaOverlays
.filter(({ script }) => scopeAppliesToServer(script, { game: serverGame, map: serverMap })) .filter(({ script }) => scopeAppliesToServer(script, { game: serverGame, map: serverMap }))
.map(({ script, version }) => ({ id: script.id, text: version.content.toString('utf8') })); .map(({ script, version }) => ({ id: script.id, text: getVersionContent(version.id).toString('utf8') }));
const composed = composeServerCfg(baselineText, [...promotedOverlays, ...inBetaScopedOverlays], serverGame, serverMap, getServer(serverName)?.map_slug); const composed = composeServerCfg(baselineText, [...promotedOverlays, ...inBetaScopedOverlays], serverGame, serverMap, getServer(serverName)?.map_slug);
return { text: composed, baselineScriptId }; return { text: composed, baselineScriptId };
@@ -604,7 +605,7 @@ function revisionFileChanges(betaId: number, name: string): Parameters<typeof qu
const server = getServer(name)!; const server = getServer(name)!;
const changes: Parameters<typeof queueFileChange>[0][] = resolved.filter(row => !isCollapsingKind(row.script.kind)) const changes: Parameters<typeof queueFileChange>[0][] = resolved.filter(row => !isCollapsingKind(row.script.kind))
.map(({ script, version }) => ({ sourceKind: 'beta', sourceId: String(betaId), server: name, kind: script.kind, .map(({ script, version }) => ({ sourceKind: 'beta', sourceId: String(betaId), server: name, kind: script.kind,
relPath: script.target_rel_path, content: version.content, scriptVersionId: version.id, inventorySource: `beta:${betaId}` })); relPath: script.target_rel_path, content: getVersionContent(version.id), scriptVersionId: version.id, inventorySource: `beta:${betaId}` }));
if (resolved.some(row => isCollapsingKind(row.script.kind))) { if (resolved.some(row => isCollapsingKind(row.script.kind))) {
const composed = composeCfgForServer(name, server.game, server.map, { const composed = composeCfgForServer(name, server.game, server.map, {
inBetaBaseline: resolved.find(row => row.script.kind === 'baseline')?.version, inBetaBaseline: resolved.find(row => row.script.kind === 'baseline')?.version,
@@ -639,7 +640,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
await mkdir(archive, { recursive: true }); await mkdir(archive, { recursive: true });
for (const r of nonCollapsing) { for (const r of nonCollapsing) {
const base = posix.basename(r.script.target_rel_path); const base = posix.basename(r.script.target_rel_path);
await writeFile(join(archive, base), r.version.content); await writeFile(join(archive, base), getVersionContent(r.version.id));
} }
// Persist the requested bytes without touching any live destination. Provisioning // Persist the requested bytes without touching any live destination. Provisioning
@@ -650,7 +651,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
try { try {
const changes: Parameters<typeof queueFileChange>[0][] = nonCollapsing.map(({ script, version }) => ({ const changes: Parameters<typeof queueFileChange>[0][] = nonCollapsing.map(({ script, version }) => ({
sourceKind: 'beta' as const, sourceId: String(betaId), server: name, sourceKind: 'beta' as const, sourceId: String(betaId), server: name,
kind: script.kind, relPath: script.target_rel_path, content: version.content, kind: script.kind, relPath: script.target_rel_path, content: getVersionContent(version.id),
scriptVersionId: version.id, inventorySource: `beta:${betaId}` scriptVersionId: version.id, inventorySource: `beta:${betaId}`
})); }));
if (collapsingBaseline || collapsingOverlays.length > 0) { if (collapsingBaseline || collapsingOverlays.length > 0) {
@@ -990,7 +991,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
priorSha = snap.prior_sha256; priorSha = snap.prior_sha256;
} else if (bss.prior_version_id) { } else if (bss.prior_version_id) {
const pv = getVersion(bss.prior_version_id); const pv = getVersion(bss.prior_version_id);
if (pv) { priorContent = pv.content; priorSha = pv.sha256; } if (pv) { priorContent = getVersionContent(pv.id); priorSha = pv.sha256; }
} }
plan = { plan = {
script, version, server: name, script, version, server: name,
@@ -1121,12 +1122,12 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
p.priorSha, p.priorSha,
p.version.sha256, p.version.sha256,
p.priorContent, p.priorContent,
p.version.content, getVersionContent(p.version.id),
betaServerSet.has(p.server) ? 0 : 1 betaServerSet.has(p.server) ? 0 : 1
); );
if (plansToActivate.has(p)) { if (plansToActivate.has(p)) {
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server, queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server,
kind: p.script.kind, relPath: p.script.target_rel_path, content: p.version.content, kind: p.script.kind, relPath: p.script.target_rel_path, content: getVersionContent(p.version.id),
scriptVersionId: p.version.id, inventorySource: `promote:${depId}`, scriptVersionId: p.version.id, inventorySource: `promote:${depId}`,
deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(p.server) }); deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(p.server) });
} }
+2 -1
View File
@@ -1,3 +1,4 @@
import { getVersionContent } from './scripts.js';
import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js'; import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js';
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js'; import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
@@ -275,7 +276,7 @@ async function resolveManagedEntry(server: string, entry: DriftEntry): Promise<B
} }
if (!entry.scriptVersionId) return null; if (!entry.scriptVersionId) return null;
const ver = getVersion(entry.scriptVersionId); const ver = getVersion(entry.scriptVersionId);
return ver?.content ?? null; return ver ? getVersionContent(ver.id) : null;
} }
function composeExpectedCfg( function composeExpectedCfg(
+46
View File
@@ -0,0 +1,46 @@
import { db } from '../db/index.js';
import type { ArtifactKind, ScriptRow, ScriptVersionRow } from './scripts.js';
/** Fixed query count regardless of the number of artifacts or retained versions. */
export function scriptCatalog(kind?: ArtifactKind) {
const version = (alias: string) => `CASE WHEN ${alias}.id IS NOT NULL THEN json_object(
'id',${alias}.id,'version',${alias}.version,'release_version',${alias}.release_version,
'sha256',${alias}.sha256,'size',${alias}.size,'state',${alias}.state,'uploaded_at',${alias}.uploaded_at) END`;
const rows = db().prepare(`SELECT s.*, ${version('c')} current_json, ${version('v')} latest_json,
COALESCE(counts.n,0) versionCount, active.beta_id runningBetaId
FROM scripts s
LEFT JOIN (SELECT script_id, COUNT(*) n, MAX(version) latest FROM script_versions GROUP BY script_id) counts ON counts.script_id=s.id
LEFT JOIN script_versions v ON v.script_id=s.id AND v.version=counts.latest
LEFT JOIN script_versions c ON c.id=s.current_version_id
LEFT JOIN (SELECT bs.script_id, MIN(b.id) beta_id FROM beta_scripts bs JOIN betas b ON b.id=bs.beta_id
WHERE b.state IN ('running','ready-promote') GROUP BY bs.script_id) active ON active.script_id=s.id
${kind ? 'WHERE s.kind=?' : ''} ORDER BY s.name`).all(...(kind ? [kind] : [])) as Array<ScriptRow & {
current_json: string | null; latest_json: string | null; versionCount: number; runningBetaId: number | null;
}>;
const servers = db().prepare('SELECT name, game, map FROM servers ORDER BY name').all() as { name: string; game: string | null; map: string | null }[];
return rows.map(({ current_json, latest_json, ...row }) => ({ ...row,
currentVersion: current_json ? JSON.parse(current_json) as ScriptVersionRow : null,
latestVersion: latest_json ? JSON.parse(latest_json) as ScriptVersionRow : null,
servers: row.scope_type && row.scope_value ? servers.filter(server => server[row.scope_type!] === row.scope_value).map(server => server.name) : []
}));
}
export function betaScriptMetadata(ids: number[]) {
const rows = db().prepare(`SELECT bs.beta_id, bs.script_id id, COALESCE(s.name,'(deleted)') name,
COALESCE(s.target_rel_path,'') target_rel_path, COALESCE(s.kind,bs.kind,'script') kind,
v.id version_id, v.version, v.release_version, v.sha256
FROM beta_scripts bs LEFT JOIN scripts s ON s.id=bs.script_id
LEFT JOIN script_versions v ON v.id=bs.script_version_id
WHERE bs.beta_id IN (SELECT value FROM json_each(?)) ORDER BY bs.script_id`).all(JSON.stringify(ids)) as Array<{
beta_id: number; id: number; name: string; target_rel_path: string; kind: ArtifactKind;
version_id: number | null; version: number; release_version: string | null; sha256: string;
}>;
const grouped = new Map<number, Array<{ id: number; name: string; target_rel_path: string; kind: ArtifactKind;
version: { id: number; version: number; release_version: string | null; sha256: string } | null }>>();
for (const { beta_id, version_id, version, release_version, sha256, ...script } of rows) {
const group = grouped.get(beta_id) ?? [];
group.push({ ...script, version: version_id === null ? null : { id: version_id, version, release_version, sha256 } });
grouped.set(beta_id, group);
}
return grouped;
}
+5 -2
View File
@@ -1,4 +1,4 @@
import { getVersion, getVersionMetadata, listScripts } from './scripts.js'; import { getVersion, getVersionContent, getVersionMetadata, listScripts } from './scripts.js';
function selectedArtifacts<Version>(game: string | null, map: string | null, readVersion: (id: number) => Version | undefined) { function selectedArtifacts<Version>(game: string | null, map: string | null, readVersion: (id: number) => Version | undefined) {
return (['script', 'plugin'] as const).flatMap((kind) => return (['script', 'plugin'] as const).flatMap((kind) =>
@@ -14,7 +14,10 @@ function selectedArtifacts<Version>(game: string | null, map: string | null, rea
/** Published file artifacts shared by provisioning and drift reconciliation. */ /** Published file artifacts shared by provisioning and drift reconciliation. */
export function promotedArtifacts(game: string | null, map: string | null) { export function promotedArtifacts(game: string | null, map: string | null) {
return selectedArtifacts(game, map, getVersion); return selectedArtifacts(game, map, id => {
const version = getVersion(id);
return version ? { ...version, content: getVersionContent(id) } : undefined;
});
} }
/** Metadata uses the same promotion and scope rules without reading file content. */ /** Metadata uses the same promotion and scope rules without reading file content. */
+11 -4
View File
@@ -32,7 +32,6 @@ export interface ScriptVersionRow {
release_version: string | null; release_version: string | null;
sha256: string; sha256: string;
size: number; size: number;
content: Buffer;
notes: string | null; notes: string | null;
state: ScriptVersionState; state: ScriptVersionState;
kind: ArtifactKind; kind: ArtifactKind;
@@ -137,14 +136,22 @@ export function listPromotedOverlaysForScope(
return rows.filter((r) => !skip.has(r.id)); return rows.filter((r) => !skip.has(r.id));
} }
export const VERSION_COLUMNS = 'id, script_id, version, release_version, sha256, size, notes, state, kind, uploaded_at';
export function getVersionContent(id: number): Buffer {
const row = db().prepare('SELECT content FROM script_versions WHERE id = ?').get(id) as { content: Buffer } | undefined;
if (!row) throw new ScriptError(`no such version (id=${id})`, 'no-such-version');
return row.content;
}
export function listVersions(scriptId: number): ScriptVersionRow[] { export function listVersions(scriptId: number): ScriptVersionRow[] {
return db() return db()
.prepare('SELECT * FROM script_versions WHERE script_id = ? ORDER BY version DESC') .prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC`)
.all(scriptId) as ScriptVersionRow[]; .all(scriptId) as ScriptVersionRow[];
} }
export function getVersion(id: number): ScriptVersionRow | undefined { export function getVersion(id: number): ScriptVersionRow | undefined {
return db().prepare('SELECT * FROM script_versions WHERE id = ?').get(id) as return db().prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE id = ?`).get(id) as
| ScriptVersionRow | ScriptVersionRow
| undefined; | undefined;
} }
@@ -162,7 +169,7 @@ export function getVersionMetadata(id: number): ScriptVersionMetadata | undefine
// upload, whether or not it has been promoted (current_version_id). // upload, whether or not it has been promoted (current_version_id).
export function getLatestVersion(scriptId: number): ScriptVersionRow | undefined { export function getLatestVersion(scriptId: number): ScriptVersionRow | undefined {
return db() return db()
.prepare('SELECT * FROM script_versions WHERE script_id = ? ORDER BY version DESC LIMIT 1') .prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC LIMIT 1`)
.get(scriptId) as ScriptVersionRow | undefined; .get(scriptId) as ScriptVersionRow | undefined;
} }
+3 -2
View File
@@ -1,3 +1,4 @@
import { getVersionContent } from './scripts.js';
import { composeServerCfg } from './config-compose.js'; import { composeServerCfg } from './config-compose.js';
import { getBaselineForScope, getVersion, listPromotedOverlaysForScope } from './scripts.js'; import { getBaselineForScope, getVersion, listPromotedOverlaysForScope } from './scripts.js';
import { selectedZombieMap, mapRotation } from './map-aliases.js'; import { selectedZombieMap, mapRotation } from './map-aliases.js';
@@ -23,7 +24,7 @@ export function getPromotedServerConfig(game: string | null, map: string | null,
...(map ? listPromotedOverlaysForScope('map', map) : []) ...(map ? listPromotedOverlaysForScope('map', map) : [])
].flatMap((overlay) => { ].flatMap((overlay) => {
const current = overlay.current_version_id ? getVersion(overlay.current_version_id) : undefined; const current = overlay.current_version_id ? getVersion(overlay.current_version_id) : undefined;
return current ? [{ id: overlay.id, text: current.content.toString('utf8') }] : []; return current ? [{ id: overlay.id, text: getVersionContent(current.id).toString('utf8') }] : [];
}); });
return Buffer.from(composeServerCfg(version.content.toString('utf8'), overlays, game, map, mapSlug), 'utf8'); return Buffer.from(composeServerCfg(getVersionContent(version.id).toString('utf8'), overlays, game, map, mapSlug), 'utf8');
} }
+17 -22
View File
@@ -1,16 +1,15 @@
import { json } from '@sveltejs/kit'; import { json } from '@sveltejs/kit';
import { import {
listBetas, listBetas,
listBetaServers,
listBetaScripts,
openBeta, openBeta,
BetaError BetaError
} from '$lib/server/fleet/beta.js'; } from '$lib/server/fleet/beta.js';
import { getScript, getVersion } from '$lib/server/fleet/scripts.js'; import { betaScriptMetadata } from '$lib/server/fleet/library-metadata.js';
import { db } from '$lib/server/db/index.js';
import type { BetaServerRow } from '$lib/server/fleet/beta.js';
import type { StoredPlan } from '$lib/server/fleet/beta-tests.js';
import { kick } from '$lib/server/pipeline/worker.js'; import { kick } from '$lib/server/pipeline/worker.js';
import { parseBetaPlan, type BetaPlan } from '$lib/beta-plan.js'; import { parseBetaPlan, type BetaPlan } from '$lib/beta-plan.js';
import { storedPlan } from '$lib/server/fleet/beta-tests.js';
import { currentBetaRevision } from '$lib/server/fleet/beta-revisions.js';
export async function GET({ url }: { url: URL }) { export async function GET({ url }: { url: URL }) {
const stateFilter = url.searchParams.get('state') as const stateFilter = url.searchParams.get('state') as
@@ -19,29 +18,25 @@ export async function GET({ url }: { url: URL }) {
| 'promoted' | 'promoted'
| 'rolled-back' | 'rolled-back'
| null; | null;
const rows = listBetas(stateFilter ?? undefined).map((b) => { const betas = listBetas(stateFilter ?? undefined);
const scripts = listBetaScripts(b.id).map((bs) => { const ids = JSON.stringify(betas.map(beta => beta.id));
const script = getScript(bs.script_id); const bundles = betaScriptMetadata(betas.map(beta => beta.id));
const version = getVersion(bs.script_version_id); const plans = new Map((db().prepare(`SELECT p.*, COALESCE((SELECT started_at FROM beta_revisions r
return { WHERE r.beta_id=p.beta_id ORDER BY revision DESC LIMIT 1), b.opened_at) started_at
id: bs.script_id, FROM beta_test_plans p JOIN betas b ON b.id=p.beta_id
name: script?.name ?? '(deleted)', WHERE p.beta_id IN (SELECT value FROM json_each(?))`).all(ids) as Array<StoredPlan & { started_at: string }>).map(plan => [plan.beta_id, plan]));
target_rel_path: script?.target_rel_path ?? '', const members = db().prepare('SELECT * FROM beta_servers WHERE beta_id IN (SELECT value FROM json_each(?)) ORDER BY server').all(ids) as BetaServerRow[];
kind: script?.kind ?? bs.kind ?? 'script', const rows = betas.map(b => {
version: version const scripts = bundles.get(b.id) ?? [];
? { id: version.id, version: version.version, release_version: version.release_version, sha256: version.sha256 } const test = plans.get(b.id);
: null
};
});
const test = storedPlan(b.id);
const plan = test ? parseBetaPlan(JSON.parse(test.plan)) : null; const plan = test ? parseBetaPlan(JSON.parse(test.plan)) : null;
return { return {
...b, ...b,
testOutcome: test?.outcome ?? null, testOutcome: test?.outcome ?? null,
testReason: test?.reason ?? null, testReason: test?.reason ?? null,
deadlineAt: plan ? new Date(Date.parse(currentBetaRevision(b.id).started_at) + plan.timeoutHours * 3600000).toISOString() : null, deadlineAt: plan ? new Date(Date.parse(test!.started_at) + plan.timeoutHours * 3600000).toISOString() : null,
scripts, scripts,
servers: listBetaServers(b.id) servers: members.filter(member => member.beta_id === b.id)
}; };
}); });
return json({ betas: rows }); return json({ betas: rows });
+3 -15
View File
@@ -1,7 +1,7 @@
import { json } from '@sveltejs/kit'; import { json } from '@sveltejs/kit';
import { db } from '$lib/server/db/index.js'; import { db } from '$lib/server/db/index.js';
import { getBeta, listBetaServers, listBetaScripts } from '$lib/server/fleet/beta.js'; import { getBeta, listBetaServers } from '$lib/server/fleet/beta.js';
import { getScript, getVersion } from '$lib/server/fleet/scripts.js'; import { betaScriptMetadata } from '$lib/server/fleet/library-metadata.js';
import { testView } from '$lib/server/fleet/beta-tests.js'; import { testView } from '$lib/server/fleet/beta-tests.js';
import { betaRevisionHistory, currentBetaRevision } from '$lib/server/fleet/beta-revisions.js'; import { betaRevisionHistory, currentBetaRevision } from '$lib/server/fleet/beta-revisions.js';
@@ -16,19 +16,7 @@ export async function GET({ params }: { params: { id: string } }) {
const beta = getBeta(id); const beta = getBeta(id);
if (!beta) return json({ ok: false, error: 'not found' }, { status: 404 }); if (!beta) return json({ ok: false, error: 'not found' }, { status: 404 });
const scripts = listBetaScripts(id).map((bs) => { const scripts = betaScriptMetadata([id]).get(id) ?? [];
const script = getScript(bs.script_id);
const version = getVersion(bs.script_version_id);
return {
id: bs.script_id,
name: script?.name ?? '(deleted)',
target_rel_path: script?.target_rel_path ?? '',
kind: script?.kind ?? bs.kind ?? 'script',
version: version
? { id: version.id, version: version.version, release_version: version.release_version, sha256: version.sha256 }
: null
};
});
const servers = listBetaServers(id); const servers = listBetaServers(id);
const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id); const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id);
+2 -1
View File
@@ -1,3 +1,4 @@
import { getVersionContent } from '$lib/server/fleet/scripts.js';
import { json } from '@sveltejs/kit'; import { json } from '@sveltejs/kit';
import { getScript, getVersion, getLatestVersion } from '$lib/server/fleet/scripts.js'; import { getScript, getVersion, getLatestVersion } from '$lib/server/fleet/scripts.js';
@@ -9,5 +10,5 @@ export async function GET({ params, url }: { params: { id: string }; url: URL })
const requested = url.searchParams.get('version'); const requested = url.searchParams.get('version');
const version = requested === null ? getLatestVersion(id) : getVersion(Number(requested)); const version = requested === null ? getLatestVersion(id) : getVersion(Number(requested));
if (!version || version.script_id !== id) return json({ error: 'Config version not found.' }, { status: 404 }); if (!version || version.script_id !== id) return json({ error: 'Config version not found.' }, { status: 404 });
return json({ text: version.content.toString('utf8'), versionId: version.id, version: version.version }, { headers: { 'Cache-Control': 'no-store' } }); return json({ text: getVersionContent(version.id).toString('utf8'), versionId: version.id, version: version.version }, { headers: { 'Cache-Control': 'no-store' } });
} }
+2 -1
View File
@@ -1,3 +1,4 @@
import { getVersionContent } from '$lib/server/fleet/scripts.js';
import { json } from '@sveltejs/kit'; import { json } from '@sveltejs/kit';
import { getServer, listServers } from '$lib/server/fleet/servers.js'; import { getServer, listServers } from '$lib/server/fleet/servers.js';
import { getBaselineForScope, getScript, getVersion, listPromotedOverlaysForScope, type ScriptRow } from '$lib/server/fleet/scripts.js'; import { getBaselineForScope, getScript, getVersion, listPromotedOverlaysForScope, type ScriptRow } from '$lib/server/fleet/scripts.js';
@@ -20,7 +21,7 @@ function inspect(serverName: string, draft?: { artifactId: number; text: string
if (!isDraft && !version) return null; if (!isDraft && !version) return null;
return { return {
source: { id: row.id, name: row.name, version: isDraft ? null : version!.version, kind: row.kind as 'baseline' | 'config', draft: isDraft }, source: { id: row.id, name: row.name, version: isDraft ? null : version!.version, kind: row.kind as 'baseline' | 'config', draft: isDraft },
text: isDraft ? draft!.text : version!.content.toString('utf8') text: isDraft ? draft!.text : getVersionContent(version!.id).toString('utf8')
}; };
} }
const base = baseline ? layer(baseline) : null; const base = baseline ? layer(baseline) : null;
+2 -34
View File
@@ -2,14 +2,11 @@ import { json } from '@sveltejs/kit';
import { import {
createScript, createScript,
isArtifactKind, isArtifactKind,
listScripts,
listVersions,
serversMatchingScope,
ScriptError, ScriptError,
type ArtifactKind, type ArtifactKind,
type ScopeType type ScopeType
} from '$lib/server/fleet/scripts.js'; } from '$lib/server/fleet/scripts.js';
import { findRunningBetaForScript } from '$lib/server/fleet/beta.js'; import { scriptCatalog } from '$lib/server/fleet/library-metadata.js';
function scopeTypeOrNull(v: FormDataEntryValue | null): ScopeType | null { function scopeTypeOrNull(v: FormDataEntryValue | null): ScopeType | null {
if (v === null) return null; if (v === null) return null;
@@ -29,36 +26,7 @@ function kindOrDefault(v: FormDataEntryValue | string | null, fallback: Artifact
export async function GET({ url }: { url: URL }) { export async function GET({ url }: { url: URL }) {
const kindParam = url.searchParams.get('kind'); const kindParam = url.searchParams.get('kind');
const kindFilter = kindParam ? (isArtifactKind(kindParam) ? kindParam : null) : null; const kindFilter = kindParam ? (isArtifactKind(kindParam) ? kindParam : null) : null;
const rows = kindFilter ? listScripts(kindFilter) : listScripts(); const scripts = scriptCatalog(kindFilter ?? undefined);
const scripts = rows.map((s) => {
const versions = listVersions(s.id);
const matched = s.scope_type && s.scope_value
? serversMatchingScope(s.scope_type as ScopeType, s.scope_value)
: [];
const runningBeta = findRunningBetaForScript(s.id);
const current = versions.find(v => v.id === s.current_version_id);
return {
...s,
currentVersion: current ? {
id: current.id, version: current.version, sha256: current.sha256,
size: current.size, state: current.state, uploaded_at: current.uploaded_at
} : null,
latestVersion: versions[0]
? {
id: versions[0].id,
version: versions[0].version,
release_version: versions[0].release_version,
sha256: versions[0].sha256,
size: versions[0].size,
state: versions[0].state,
uploaded_at: versions[0].uploaded_at
}
: null,
versionCount: versions.length,
servers: matched,
runningBetaId: runningBeta?.id ?? null
};
});
return json({ scripts }); return json({ scripts });
} }
+1 -1
View File
@@ -154,7 +154,7 @@ describe('beta revisions', () => {
it.each(['stale', 'identical', 'empty', 'unbundled', 'duplicate', 'closed', 'deleted', 'interrupted', 'activating'])('rejects %s updates without saving a revision', async reason => { it.each(['stale', 'identical', 'empty', 'unbundled', 'duplicate', 'closed', 'deleted', 'interrupted', 'activating'])('rejects %s updates without saving a revision', async reason => {
const x = await setup(); const x = await setup();
const upload = { scriptId: x.script.id, content: Buffer.from('changed') }; const upload = { scriptId: x.script.id, content: Buffer.from('changed') };
if (reason === 'identical') upload.content = Buffer.from(x.version.content); if (reason === 'identical') upload.content = Buffer.from((await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(x.version.id));
if (reason === 'empty') upload.content = Buffer.alloc(0); if (reason === 'empty') upload.content = Buffer.alloc(0);
if (reason === 'unbundled') upload.scriptId = 999; if (reason === 'unbundled') upload.scriptId = 999;
if (reason === 'closed') x.db.prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(x.betaId); if (reason === 'closed') x.db.prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(x.betaId);
+1 -1
View File
@@ -35,7 +35,7 @@ it('covers every foreign key child prefix, including lazy host and library schem
const prefixes = indexes.filter(index => !index.partial).map(index => const prefixes = indexes.filter(index => !index.partial).map(index =>
(handle.pragma(`index_info(${index.name})`) as { name: string }[])[0]?.name); (handle.pragma(`index_info(${index.name})`) as { name: string }[])[0]?.name);
const columns = handle.pragma(`table_info(${name})`) as { name: string; type: string; pk: number }[]; const columns = handle.pragma(`table_info(${name})`) as { name: string; type: string; pk: number }[];
if (columns.filter(column => column.pk).length === 1) prefixes.push(columns.find(column => column.pk)?.name); if (columns.filter(column => column.pk).length === 1) prefixes.push(columns.find(column => column.pk)!.name);
for (const key of foreignKeys.filter(key => key.seq === 0)) { for (const key of foreignKeys.filter(key => key.seq === 0)) {
if (!prefixes.includes(key.from)) missing.push(`${name}.${key.from}`); if (!prefixes.includes(key.from)) missing.push(`${name}.${key.from}`);
} }
+1 -1
View File
@@ -378,7 +378,7 @@ describe('drift rebuild campaigns', () => {
expect(await readFile(stray, 'utf8')).toBe('untracked'); expect(await readFile(stray, 'utf8')).toBe('untracked');
expect(db().prepare('SELECT rel_path, verb, content_before, content_after FROM deployment_files WHERE deployment_id = ? ORDER BY rel_path').all(campaignId)) expect(db().prepare('SELECT rel_path, verb, content_before, content_after FROM deployment_files WHERE deployment_id = ? ORDER BY rel_path').all(campaignId))
.toEqual([ .toEqual([
{ rel_path: 'managed.gsc', verb: 'updated', content_before: Buffer.from('old content'), content_after: version.content }, { rel_path: 'managed.gsc', verb: 'updated', content_before: Buffer.from('old content'), content_after: (await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id) },
{ rel_path: 'stray.cfg', verb: 'removed', content_before: Buffer.from('untracked'), content_after: null } { rel_path: 'stray.cfg', verb: 'removed', content_before: Buffer.from('untracked'), content_after: null }
]); ]);
await sweep(); await sweep();
+53
View File
@@ -0,0 +1,53 @@
import Database from 'better-sqlite3';
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { migrateVersionLayout } from '../../src/lib/server/db/version-layout.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
it('returns metadata without plugin bytes and lists many artifacts in two queries', async () => {
const { createScript, getVersion, getVersionContent, getLatestVersion, listVersions } = await import('../../src/lib/server/fleet/scripts.js');
const { scriptCatalog, betaScriptMetadata } = await import('../../src/lib/server/fleet/library-metadata.js');
const { db } = await import('../../src/lib/server/db/index.js');
const content = Buffer.alloc(1024 * 1024, 77);
let id = 0;
for (let i = 0; i < 20; i++) id = createScript({ name: `plugin-${i}`, kind: 'plugin', targetRelPath: 'plugin.dll', content }).version.id;
expect(getVersionContent(id)).toEqual(content);
expect(getVersion(id)).not.toHaveProperty('content');
expect(getLatestVersion(getVersion(id)!.script_id)).not.toHaveProperty('content');
expect(listVersions(getVersion(id)!.script_id)[0]).not.toHaveProperty('content');
const prepare = vi.spyOn(db(), 'prepare');
expect(scriptCatalog('plugin')).toHaveLength(20);
expect(prepare).toHaveBeenCalledTimes(2);
betaScriptMetadata([]);
expect(prepare.mock.calls.every(([sql]) => !/\bcontent\b/.test(sql))).toBe(true);
});
it('moves legacy content last without losing bytes, IDs, indexes, triggers or references', () => {
const handle = new Database(':memory:');
try {
handle.exec(`PRAGMA foreign_keys=ON;
CREATE TABLE script_versions(id INTEGER PRIMARY KEY AUTOINCREMENT, script_id INTEGER,
version INTEGER, content BLOB NOT NULL, state TEXT, release_version TEXT, UNIQUE(script_id,version));
CREATE INDEX version_state ON script_versions(state);
CREATE TABLE child(version INTEGER REFERENCES script_versions(id));
CREATE TABLE updates(n INTEGER); INSERT INTO updates VALUES(0);
CREATE TRIGGER version_update AFTER UPDATE ON script_versions BEGIN UPDATE updates SET n=n+1; END;
INSERT INTO script_versions VALUES(7,1,1,x'00ff','tested','v1');
INSERT INTO child VALUES(7);
INSERT INTO script_versions VALUES(100,1,2,x'aa','untested',NULL);
DELETE FROM script_versions WHERE id=100;`);
const before = handle.prepare('SELECT * FROM script_versions').all();
migrateVersionLayout(handle);
migrateVersionLayout(handle);
expect(handle.prepare('SELECT * FROM script_versions').all()).toEqual(before);
expect((handle.pragma('table_info(script_versions)') as { name: string }[]).at(-1)?.name).toBe('content');
expect(handle.pragma('foreign_key_check')).toEqual([]);
expect(handle.prepare("INSERT INTO script_versions(script_id,version,content) VALUES(1,3,x'01')").run().lastInsertRowid).toBe(101);
handle.prepare("UPDATE script_versions SET state='tested' WHERE id=101").run();
expect(handle.prepare('SELECT n FROM updates').get()).toEqual({ n: 1 });
expect((handle.pragma('index_list(script_versions)') as { name: string }[]).some(index => index.name === 'version_state')).toBe(true);
} finally { handle.close(); }
});
+1 -1
View File
@@ -40,7 +40,7 @@ async function setup(capable = true) {
const baseline = promoted('baseline', 'Baseline', Buffer.from('set sv_voice "1"\n')); const baseline = promoted('baseline', 'Baseline', Buffer.from('set sv_voice "1"\n'));
for (const [root, artifact] of [[scriptRoot, script], [pluginRoot, plugin]] as const) { for (const [root, artifact] of [[scriptRoot, script], [pluginRoot, plugin]] as const) {
const { version } = artifact; const { version } = artifact;
contents.set(`${root}/${artifact.script.target_rel_path}`, version.content); contents.set(`${root}/${artifact.script.target_rel_path}`, (await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id));
recordScript(server, artifact.script.target_rel_path, version.id, version.sha256, version.size, 'test'); recordScript(server, artifact.script.target_rel_path, version.id, version.sha256, version.size, 'test');
} }
const { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } = await import('../../src/lib/server/fleet/server-config.js'); const { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } = await import('../../src/lib/server/fleet/server-config.js');
+3 -2
View File
@@ -26,7 +26,7 @@ describe('plugin release versions', () => {
[3, 'v3.0.4'], [2, 'v3.0.4'], [1, 'v3.1.0'] [3, 'v3.0.4'], [2, 'v3.0.4'], [1, 'v3.1.0']
]); ]);
expect(older.sha256).not.toBe(rebuild.sha256); expect(older.sha256).not.toBe(rebuild.sha256);
expect(getVersion(version.id)!.content).toEqual(plugin.content); expect((await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id)).toEqual(plugin.content);
}); });
it('allows unlabeled uploads and non-semver release tags', async () => { it('allows unlabeled uploads and non-semver release tags', async () => {
@@ -53,8 +53,9 @@ describe('plugin release versions', () => {
db().exec('ALTER TABLE script_versions DROP COLUMN release_version'); db().exec('ALTER TABLE script_versions DROP COLUMN release_version');
closeDb(); closeDb();
expect(getVersion(version.id)).toMatchObject({ expect(getVersion(version.id)).toMatchObject({
id: version.id, version: 1, release_version: null, sha256: version.sha256, content: plugin.content id: version.id, version: 1, release_version: null, sha256: version.sha256
}); });
expect((await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id)).toEqual(plugin.content);
closeDb(); closeDb();
expect(getVersion(version.id)!.release_version).toBeNull(); expect(getVersion(version.id)!.release_version).toBeNull();
}); });