242 lines
14 KiB
JavaScript
242 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Benchmark actual leaderboard modules against deterministic, isolated SQLite histories.
|
|
* node scripts/benchmark-leaderboard.mjs --scenario all --baseline-ref <commit> --runs 3
|
|
* Uses a TypeScript module loader to replace only db(); it never opens the application DB.
|
|
*/
|
|
import assert from 'node:assert/strict';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { createHash } from 'node:crypto';
|
|
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
import { createRequire } from 'node:module';
|
|
import { arch, cpus, platform, tmpdir } from 'node:os';
|
|
import { dirname, join, relative, resolve } from 'node:path';
|
|
import { performance } from 'node:perf_hooks';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
const require = createRequire(join(repo, 'package.json'));
|
|
const Database = require('better-sqlite3');
|
|
const ts = require('typescript');
|
|
const arguments_ = process.argv.slice(2);
|
|
const options = { scenario: 'small', runs: 3 };
|
|
const allowed = new Set(['--scenario', '--runs', '--baseline-ref', '--json']);
|
|
for (let index = 0; index < arguments_.length; index += 2) {
|
|
const key = arguments_[index];
|
|
const value = arguments_[index + 1];
|
|
if (!allowed.has(key) || !value) throw new Error('Usage: --scenario small|large|all --runs N --baseline-ref COMMIT --json PATH');
|
|
options[key.slice(2)] = value;
|
|
}
|
|
options.runs = Number(options.runs);
|
|
if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 20) throw new Error('--runs must be an integer from 1 to 20');
|
|
if (!['small', 'large', 'all'].includes(options.scenario)) throw new Error('--scenario must be small, large, or all');
|
|
const baseline = options['baseline-ref']
|
|
? execFileSync('git', ['rev-parse', '--verify', '--end-of-options', `${options['baseline-ref']}^{commit}`], { cwd: repo, encoding: 'utf8' }).trim()
|
|
: null;
|
|
const definitions = [
|
|
{ name: 'small', matches: 2000, players: 500 },
|
|
{ name: 'large', matches: 50000, players: 10000 }
|
|
].filter(item => options.scenario === 'all' || item.name === options.scenario);
|
|
const temporary = mkdtempSync(join(tmpdir(), 'cherry-leaderboard-benchmark-'));
|
|
const source = (file, ref) => ref
|
|
? execFileSync('git', ['show', `${ref}:${file}`], { cwd: repo, encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 })
|
|
: readFileSync(join(repo, file), 'utf8');
|
|
const percentile = (values, fraction) => [...values].sort((a, b) => a - b)[Math.ceil(values.length * fraction) - 1];
|
|
|
|
function seed(handle, definition) {
|
|
handle.exec(source('src/lib/server/db/schema.sql'));
|
|
const match = handle.prepare(`INSERT INTO leaderboard_matches
|
|
(id, server, game, map, map_key, map_name, started_at, ended_at, last_received_at,
|
|
status, start_observed, start_players, highest_round, last_event, last_fingerprint)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 4, ?, ?, ?)`);
|
|
const identity = handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
|
|
VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET
|
|
name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END,
|
|
last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`);
|
|
const participation = handle.prepare(`INSERT INTO leaderboard_match_players
|
|
(match_id, guid, highest_round, rounds_survived, kills, headshots, downs, revives, best_score,
|
|
last_round, present, has_stats, completed, first_seen_at, last_seen_at,
|
|
first_observed_round, present_at_start)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
let state = 20260918;
|
|
const random = () => { state = (Math.imul(state, 1664525) + 1013904223) >>> 0; return state / 4294967296; };
|
|
handle.transaction(() => {
|
|
for (let id = 1; id <= definition.matches; id++) {
|
|
const mapIndex = Math.floor(random() * 12);
|
|
const game = mapIndex < 6 ? 't5' : 't6';
|
|
const map = `map_${mapIndex}`;
|
|
const status = id % 20 === 0 ? 'active' : id % 20 === 1 ? 'interrupted' : 'completed';
|
|
const started = new Date(Date.UTC(2023, 0, 1) + Math.floor(id / definition.matches * 3 * 365 * 86400000)).toISOString();
|
|
const ended = new Date(Date.parse(started) + (20 + Math.floor(random() * 130)) * 60000).toISOString();
|
|
const round = id % 101 === 0 ? 1 : 2 + Math.floor(random() * 60);
|
|
match.run(id, `server_${id % 24}`, game, map, `${game}:${map}`, `Map ${mapIndex}`,
|
|
started, status === 'completed' ? ended : null, ended, status, round,
|
|
status === 'completed' ? 'game_end' : 'heartbeat', `event_${id}`);
|
|
const roster = new Set();
|
|
while (roster.size < 4) roster.add(Math.floor(random() ** 2 * definition.players));
|
|
// Reserve one completed participation for an explicitly ineligible player.
|
|
if (id === definition.matches - 2) roster.delete([...roster][3]);
|
|
for (const player of roster) {
|
|
const guid = String(10000000000000000n + BigInt(player));
|
|
const name = `${game === 't6' ? 'Alias' : 'Player'} ${String(player).padStart(5, '0')}`;
|
|
identity.run(game, guid, name, started, ended);
|
|
const kills = Math.floor(round * 18 + random() * 800);
|
|
const presentAtStart = random() > 0.1;
|
|
const firstRound = presentAtStart ? 0 : Math.max(1, Math.floor(random() * round));
|
|
participation.run(id, guid, round, Math.max(0, round - Math.max(1, firstRound)), kills,
|
|
Math.floor(kills * random()), Math.floor(random() * 8), Math.floor(random() * 12),
|
|
Math.floor(random() * 100000), Math.max(0, round - 1), Number(status === 'active'),
|
|
Number(random() > 0.01), Number(status === 'completed' && random() > 0.15),
|
|
started, ended, firstRound, Number(presentAtStart));
|
|
}
|
|
if (id === definition.matches - 2) {
|
|
identity.run(game, 'benchmark-low-sample', 'Low sample', started, ended);
|
|
participation.run(id, 'benchmark-low-sample', 2, 1, 20, 5, 0, 0, 1000,
|
|
1, 0, 1, 1, started, ended, 0, 1);
|
|
}
|
|
}
|
|
})();
|
|
handle.pragma('wal_checkpoint(TRUNCATE)');
|
|
}
|
|
|
|
function loadImplementation(handle, ref) {
|
|
const modules = new Map();
|
|
const hashes = {};
|
|
let recorded = null;
|
|
const database = { prepare(sql) {
|
|
const statement = handle.prepare(sql);
|
|
return Object.fromEntries(['all', 'get'].map(method => [method, (...parameters) => {
|
|
const start = performance.now();
|
|
const result = statement[method](...parameters);
|
|
if (recorded) recorded.push({ sql, parameters, milliseconds: performance.now() - start });
|
|
return result;
|
|
}]));
|
|
} };
|
|
function load(file) {
|
|
if (file === 'src/lib/server/db/index.ts') return { db: () => database };
|
|
if (modules.has(file)) return modules.get(file);
|
|
const text = source(file, ref);
|
|
hashes[file] = createHash('sha256').update(text).digest('hex');
|
|
const compiled = ts.transpileModule(text, {
|
|
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }
|
|
}).outputText;
|
|
const exports = {};
|
|
modules.set(file, exports);
|
|
const localRequire = specifier => {
|
|
if (!specifier.startsWith('.')) return require(specifier);
|
|
const target = relative(repo, resolve(repo, dirname(file), specifier)).replace(/\.js$/, '.ts');
|
|
if (!target.startsWith('src/lib/')) throw new Error(`Unexpected benchmark dependency: ${target}`);
|
|
return load(target);
|
|
};
|
|
new Function('require', 'exports', compiled)(localRequire, exports);
|
|
return exports;
|
|
}
|
|
const api = load('src/lib/server/fleet/leaderboard.ts');
|
|
return {
|
|
name: ref ? 'baseline' : 'current', ref, hashes,
|
|
metrics: load('src/lib/leaderboard.ts').leaderboardMetrics.map(metric => metric.key),
|
|
run(test) {
|
|
return test.guid ? api.getPlayerLeaderboardEntry(test.guid, test.options) : api.getLeaderboard(test.options);
|
|
},
|
|
measure(test) {
|
|
recorded = [];
|
|
const start = performance.now();
|
|
const result = this.run(test);
|
|
const milliseconds = performance.now() - start;
|
|
const statements = recorded;
|
|
recorded = null;
|
|
return { result, milliseconds, statements };
|
|
}
|
|
};
|
|
}
|
|
|
|
const report = {
|
|
environment: { node: process.version, sqlite: null, platform: platform(), arch: arch(), cpu: cpus()[0]?.model },
|
|
baselineRef: baseline, runs: options.runs, seed: 20260918,
|
|
notes: 'Synthetic file-backed SQLite, one warmup per case, rotating execution order, warm OS cache. Includes SQL preparation and source-record lookup; excludes HTTP and browser rendering. Shared local CPU; timings are not production latency. Schema and indexes are current for both implementations.',
|
|
scenarios: []
|
|
};
|
|
try {
|
|
for (const definition of definitions) {
|
|
const handle = new Database(join(temporary, `${definition.name}.sqlite`));
|
|
try {
|
|
report.environment.sqlite = handle.prepare('SELECT sqlite_version() AS version').get().version;
|
|
seed(handle, definition);
|
|
const implementations = [...(baseline ? [loadImplementation(handle, baseline)] : []), loadImplementation(handle)];
|
|
const cases = [
|
|
{ name: 'unfiltered', options: {} },
|
|
{ name: 'kills', options: { metric: 'kills' } },
|
|
{ name: 'game', options: { game: 't5', metric: 'kills' } },
|
|
{ name: 'map', options: { game: 't6', map: 't6:map_8', metric: 'kills' } },
|
|
{ name: 'search', options: { q: '00042', metric: 'kills' } },
|
|
{ name: 'deep-page', options: { metric: 'kills', offset: Math.floor(definition.players / 2) } },
|
|
{ name: 'efficiency', options: { metric: 'headshotRate' } },
|
|
{ name: 'exact-player', guid: '10000000000000042', options: { metric: 'kills' } },
|
|
{ name: 'empty-search', options: { q: 'No such player exists' } },
|
|
{ name: 'past-end', options: { offset: 999999 } },
|
|
{ name: 'empty-map', options: { map: 't5:missing' } }
|
|
];
|
|
const counts = handle.prepare(`SELECT COUNT(*) AS participationRows,
|
|
COUNT(DISTINCT guid) AS uniquePlayers FROM leaderboard_match_players`).get();
|
|
const scenario = { ...definition, ...counts, maps: 12, servers: 24, completedShare: 0.9, results: [] };
|
|
report.scenarios.push(scenario);
|
|
// Broad parity is separate from timings so every ratio and edge case is
|
|
// checked without multiplying the larger fixture's timed request matrix.
|
|
if (baseline && definition.name === 'small') {
|
|
const scopes = [{}, { game: 't5' }, { game: 't6', map: 't6:map_8' }, { q: '00042' },
|
|
{ q: 'No such player exists' }, { offset: 999999 }, { map: 't5:missing' }];
|
|
let checked = 0;
|
|
for (const metric of implementations[1].metrics) {
|
|
for (const scope of scopes) {
|
|
const test = { options: { ...scope, metric } };
|
|
assert.deepStrictEqual(implementations[1].run(test), implementations[0].run(test),
|
|
`${definition.name}/${metric}/${JSON.stringify(scope)}: full-response parity`);
|
|
checked++;
|
|
}
|
|
for (const guid of ['10000000000000042', 'benchmark-low-sample', 'benchmark-missing']) {
|
|
const test = { guid, options: { metric } };
|
|
assert.deepStrictEqual(implementations[1].run(test), implementations[0].run(test),
|
|
`${definition.name}/${metric}/${guid}: exact-player parity`);
|
|
checked++;
|
|
}
|
|
}
|
|
scenario.parityCases = checked;
|
|
console.log(JSON.stringify({ scenario: definition.name, parityCases: checked, parity: true }));
|
|
}
|
|
for (const test of cases) {
|
|
const expected = implementations[0].run(test);
|
|
for (const implementation of implementations.slice(1)) {
|
|
assert.deepStrictEqual(implementation.run(test), expected, `${definition.name}/${test.name}: baseline parity`);
|
|
}
|
|
const samples = Object.fromEntries(implementations.map(item => [item.name, []]));
|
|
const latestStatements = {};
|
|
for (let index = 0; index < options.runs; index++) {
|
|
const rotation = index % implementations.length;
|
|
for (const implementation of [...implementations.slice(rotation), ...implementations.slice(0, rotation)]) {
|
|
const measured = implementation.measure(test);
|
|
samples[implementation.name].push(measured.milliseconds);
|
|
latestStatements[implementation.name] = measured.statements;
|
|
}
|
|
}
|
|
const result = { ...test, parity: baseline ? true : null, implementations: implementations.map(implementation => ({
|
|
name: implementation.name, sourceHashes: implementation.hashes,
|
|
medianMs: percentile(samples[implementation.name], 0.5),
|
|
p95Ms: percentile(samples[implementation.name], 0.95), samplesMs: samples[implementation.name],
|
|
statements: latestStatements[implementation.name].map(statement => ({ ...statement,
|
|
plan: handle.prepare(`EXPLAIN QUERY PLAN ${statement.sql}`).all(...statement.parameters)
|
|
}))
|
|
})) };
|
|
scenario.results.push(result);
|
|
console.log(JSON.stringify({ scenario: definition.name, case: test.name, parity: result.parity,
|
|
medianMs: Object.fromEntries(result.implementations.map(item => [item.name, Math.round(item.medianMs)])) }));
|
|
}
|
|
} finally { handle.close(); }
|
|
}
|
|
if (options.json) {
|
|
const target = resolve(options.json);
|
|
mkdirSync(dirname(target), { recursive: true });
|
|
writeFileSync(target, JSON.stringify(report, null, 2));
|
|
console.log(`Detailed results: ${target}`);
|
|
}
|
|
} finally { rmSync(temporary, { recursive: true, force: true }); }
|