josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Batch remote drift checks into agent file manifests

Browse Source
This commit is contained in:
josh
2026-10-05 12:52:33 -04:00
parent 10ab071bce
commit f16879036e
11 changed files with 779 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/backends/files.ts
+31 -1
View File
@@ -1,7 +1,8 @@
import * as fs from 'node:fs/promises';
import { relative, resolve, sep } from 'node:path';
import { join, posix, relative, resolve, sep } from 'node:path';
import { config } from '../config.js';
import { nativeInstance, nativeRequest } from './native.js';
import { getHost } from '../hosts/index.js';
function remote(path: string) {
for (const root of [config.serversRoot, config.t6ServersRoot]) {
@@ -16,6 +17,35 @@ function remote(path: string) {
}
function missing(path: string): never { throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' }); }
export interface FileFingerprint { size: number; sha256: string }
/** One scoped agent scan; null preserves compatibility with older agents and local files. */
export async function nativeFileManifest(root: string, directories: string[], ignoreNames: string[], managedPaths: string[]): Promise<Map<string, FileFingerprint> | null> {
const target = remote(root);
if (!target || target.path !== '.' || !getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest')) return null;
const roots = [...new Set(directories.map(directory => relative(root, directory).split(sep).join('/')))];
// Preserve legacy configurations that scan the entire instance root.
if (roots.some(directory => !directory)) return null;
const paths = [...new Set(managedPaths.map(path => relative(root, path).split(sep).join('/')))];
const result = await nativeRequest<{ files: Array<FileFingerprint & { path: string }> }>(target.instance, 'files.manifest', {
roots, paths, ignoreNames: ignoreNames.filter(name => !paths.some(path => path.split('/').includes(name)))
});
if (!result || !Array.isArray(result.files) || result.files.length > 10_000) throw new Error('Agent returned an invalid file manifest.');
const manifest = new Map<string, FileFingerprint>();
for (const entry of result.files) {
if (!entry || typeof entry.path !== 'string' || !entry.path || entry.path.includes('\\') || entry.path.includes('\0')
|| posix.normalize(entry.path) !== entry.path || entry.path.split('/').some(part => part === '..' || part === '.')
|| !roots.some(directory => entry.path.startsWith(`${directory}/`))
|| !Number.isSafeInteger(entry.size) || entry.size < 0 || !/^[a-f0-9]{64}$/.test(entry.sha256)) {
throw new Error('Agent returned an invalid file manifest entry.');
}
const path = join(root, entry.path);
if (manifest.has(path)) throw new Error('Agent returned duplicate file manifest entries.');
manifest.set(path, { size: entry.size, sha256: entry.sha256 });
}
return manifest;
}
export async function readFile(path: string): Promise<Buffer> {
const target = remote(path);
if (!target) return fs.readFile(path);
src/lib/server/backends/native.ts
+1 -1
View File
@@ -31,7 +31,7 @@ export function unplacedNativeCreation(server: string): string | null {
}
const observations = new Map<string, Promise<unknown>>();
const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'logs.read']);
const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'files.manifest', 'logs.read']);
/** Share overlapping observations only, never completed results or mutations. */
export async function nativeRequest<T = Record<string, unknown>>(instance: NativeInstance, kind: string, payload: Record<string, unknown> = {}, key?: string): Promise<T> {
src/lib/server/fleet/drift.ts
+32 -19
View File
@@ -1,4 +1,4 @@
import { readdir, readFile, stat } from '../backends/files.js';
import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js';
import { randomUUID } from 'node:crypto';
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
import { join, posix, relative, sep } from 'node:path';
@@ -11,7 +11,7 @@ import { getBaselineForScope, getVersion } from './scripts.js';
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
import { selectedZombieMap } from './map-aliases.js';
import { getServer, listServers } from './servers.js';
import { promotedArtifacts } from './promoted-artifacts.js';
import { promotedArtifactMetadata } from './promoted-artifacts.js';
import { queueFileChange } from './staged-files.js';
import { trackActivity } from '../pipeline/activity.js';
@@ -88,6 +88,12 @@ async function readExistingFile(path: string): Promise<Buffer | null> {
});
}
async function fingerprint(path: string, manifest: Map<string, FileFingerprint> | null): Promise<FileFingerprint | null> {
if (manifest) return manifest.get(path) ?? null;
const content = await readExistingFile(path);
return content === null ? null : { size: content.byteLength, sha256: sha256Buffer(content) };
}
async function walkFiles(root: string): Promise<string[]> {
const out: string[] = [];
let directories = [root];
@@ -128,11 +134,15 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv || srv.destroyed_at || hasActiveBeta(server)) return [];
// A missing mount is unavailable evidence, not an empty managed tree.
await requireServerDirectory(server);
const drift = await scanManagedForServer(server);
const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) => join(destOfKind(server, kind), script.target_rel_path));
if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id) expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server)));
if (srv.created_by && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE));
const manifest = await nativeFileManifest(serverDir(server), DRIFT_DESTS.map(kind => destOfKind(server, kind)), [...DRIFT_IGNORE_FILENAMES], expectedPaths);
if (!manifest) await requireServerDirectory(server);
const drift = await scanManagedForServer(server, manifest);
// Previous inventory is an audit of installed bytes, not permission to
// retain artifacts whose scope was moved or removed from the library.
const managed = new Set(promotedArtifacts(srv.game, srv.map)
const managed = new Set(promotedArtifactMetadata(srv.game, srv.map)
.map(({ kind, script }) => `${kind}:${script.target_rel_path}`));
const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined;
if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`);
@@ -145,7 +155,10 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
// Scan for untracked files only after resolving managed targets.
for (const kind of DRIFT_DESTS) {
const dest = destOfKind(server, kind);
const onDisk = await walkFiles(dest);
const onDisk = manifest
? [...manifest.keys()].filter(path => path.startsWith(`${dest}${sep}`)).map(path => toPosix(relative(dest, path)))
.filter(path => !path.split('/').some(name => DRIFT_IGNORE_FILENAMES.has(name))).sort()
: await walkFiles(dest);
if (onDisk.length === 0) continue;
const untracked = await scanBatch(onDisk, async (relPath): Promise<DriftEntry | null> => {
@@ -155,11 +168,11 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
&& srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) return null;
const abs = join(dest, relPath);
try {
const buf = await readFile(abs);
const info = await fingerprint(abs, manifest);
if (!info) return null;
return {
server, relPath,
size: buf.byteLength,
sha256: sha256Buffer(buf),
...info,
kind,
driftKind: 'untracked'
};
@@ -172,14 +185,14 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
drift.push(...untracked.filter((entry): entry is DriftEntry => entry !== null));
}
await requireServerDirectory(server);
if (!manifest) await requireServerDirectory(server);
return hasActiveBeta(server) ? [] : drift;
}
// Every promoted artifact whose scope matches this server's game/map should
// have its target file present on disk. Anything absent is 'missing' drift.
// Configs are handled as a composed layer (baseline + overlays → one cfg).
async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
async function scanManagedForServer(server: string, manifest: Map<string, FileFingerprint> | null): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv) return [];
const out: DriftEntry[] = [];
@@ -190,8 +203,8 @@ async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null;
if (mapConfig) {
const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE);
const content = await readExistingFile(abs);
if (!content?.equals(mapConfig)) {
const content = await fingerprint(abs, manifest);
if (content?.sha256 !== sha256Buffer(mapConfig)) {
out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing',
size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) });
}
@@ -199,10 +212,10 @@ async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
// Inventory records the last managed write, not the current disk bytes.
// Check both so edits and outdated versions are detected on tracked files.
const managed = await scanBatch(promotedArtifacts(srv.game, srv.map), async ({ kind, script, version }): Promise<DriftEntry | null> => {
const content = await readExistingFile(join(destOfKind(server, kind), script.target_rel_path));
const managed = await scanBatch(promotedArtifactMetadata(srv.game, srv.map), async ({ kind, script, version }): Promise<DriftEntry | null> => {
const content = await fingerprint(join(destOfKind(server, kind), script.target_rel_path), manifest);
const inventory = getServerScript(server, script.target_rel_path);
if (content?.equals(version.content) && inventory?.sha256 === version.sha256
if (content?.sha256 === version.sha256 && inventory?.sha256 === version.sha256
&& inventory.script_version_id === version.id) return null;
return {
server, relPath: script.target_rel_path, kind,
@@ -226,8 +239,8 @@ async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
const targetRel = configFileOf(server);
const abs = join(dest, targetRel);
const inv = getServerScript(server, targetRel);
const content = await readExistingFile(abs);
const onDiskMatches = content?.equals(composed.content) && inv?.sha256 === composed.sha256;
const content = await fingerprint(abs, manifest);
const onDiskMatches = content?.sha256 === composed.sha256 && inv?.sha256 === composed.sha256;
if (!onDiskMatches) {
out.push({
server,
@@ -366,7 +379,7 @@ export async function prepareDriftCampaign(server: string): Promise<PreparedDrif
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } };
});
await requireServerDirectory(server);
if (entries.length) await requireServerDirectory(server);
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
return { plan: buildPlan(servers, files.map(({ file }) => file)), files };
}
src/lib/server/hosts/index.ts
+1 -1
View File
@@ -435,7 +435,7 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
// Accepted/running/unknown commands keep the same immutable ID. The agent
// consults its durable journal before acknowledging or executing a replay.
const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','files.manifest','logs.read')
OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthLane = host.inventory.capabilities?.includes('operations.health-lane');