412 lines
18 KiB
JavaScript
412 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Benchmark actual leaderboard modules against deterministic, isolated SQLite histories.
|
|
* node scripts/benchmark-leaderboard.mjs --scenario all --baseline-ref <commit> --runs 3
|
|
* Uses a TypeScript module loader to replace only db(); it never opens the application DB.
|
|
*/
|
|
import assert from 'node:assert/strict';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { createHash } from 'node:crypto';
|
|
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
import { createRequire } from 'node:module';
|
|
import { arch, cpus, platform, tmpdir } from 'node:os';
|
|
import { dirname, join, relative, resolve } from 'node:path';
|
|
import { performance } from 'node:perf_hooks';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
const require = createRequire(join(repo, 'package.json'));
|
|
const Database = require('better-sqlite3');
|
|
const ts = require('typescript');
|
|
const arguments_ = process.argv.slice(2);
|
|
const options = { scenario: 'small', runs: 3 };
|
|
const allowed = new Set(['--scenario', '--runs', '--baseline-ref', '--json']);
|
|
for (let index = 0; index < arguments_.length; index += 2) {
|
|
const key = arguments_[index];
|
|
const value = arguments_[index + 1];
|
|
if (!allowed.has(key) || !value)
|
|
throw new Error(
|
|
'Usage: --scenario small|large|all --runs N --baseline-ref COMMIT --json PATH'
|
|
);
|
|
options[key.slice(2)] = value;
|
|
}
|
|
options.runs = Number(options.runs);
|
|
if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 20)
|
|
throw new Error('--runs must be an integer from 1 to 20');
|
|
if (!['small', 'large', 'all'].includes(options.scenario))
|
|
throw new Error('--scenario must be small, large, or all');
|
|
const baseline = options['baseline-ref']
|
|
? execFileSync(
|
|
'git',
|
|
['rev-parse', '--verify', '--end-of-options', `${options['baseline-ref']}^{commit}`],
|
|
{ cwd: repo, encoding: 'utf8' }
|
|
).trim()
|
|
: null;
|
|
const definitions = [
|
|
{ name: 'small', matches: 2000, players: 500 },
|
|
{ name: 'large', matches: 50000, players: 10000 }
|
|
].filter((item) => options.scenario === 'all' || item.name === options.scenario);
|
|
const temporary = mkdtempSync(join(tmpdir(), 'cherry-leaderboard-benchmark-'));
|
|
const source = (file, ref) =>
|
|
ref
|
|
? execFileSync('git', ['show', `${ref}:${file}`], {
|
|
cwd: repo,
|
|
encoding: 'utf8',
|
|
maxBuffer: 10 * 1024 * 1024
|
|
})
|
|
: readFileSync(join(repo, file), 'utf8');
|
|
const percentile = (values, fraction) =>
|
|
[...values].sort((a, b) => a - b)[Math.ceil(values.length * fraction) - 1];
|
|
|
|
function seed(handle, definition) {
|
|
handle.exec(source('src/lib/server/db/schema.sql'));
|
|
handle.exec(
|
|
"CREATE TABLE report_revisions(topic TEXT PRIMARY KEY, revision INTEGER NOT NULL); INSERT INTO report_revisions VALUES ('leaderboard',0)"
|
|
);
|
|
const match = handle.prepare(`INSERT INTO leaderboard_matches
|
|
(id, server, game, map, map_key, map_name, started_at, ended_at, last_received_at,
|
|
status, start_observed, start_players, highest_round, last_event, last_fingerprint)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 4, ?, ?, ?)`);
|
|
const identity =
|
|
handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
|
|
VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET
|
|
name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END,
|
|
last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`);
|
|
const participation = handle.prepare(`INSERT INTO leaderboard_match_players
|
|
(match_id, guid, highest_round, rounds_survived, kills, headshots, downs, revives, best_score,
|
|
last_round, present, has_stats, completed, first_seen_at, last_seen_at,
|
|
first_observed_round, present_at_start)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
let state = 20260918;
|
|
const random = () => {
|
|
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
|
|
return state / 4294967296;
|
|
};
|
|
handle.transaction(() => {
|
|
for (let id = 1; id <= definition.matches; id++) {
|
|
const mapIndex = Math.floor(random() * 12);
|
|
const game = mapIndex < 6 ? 't5' : 't6';
|
|
const map = `map_${mapIndex}`;
|
|
const status = id % 20 === 0 ? 'active' : id % 20 === 1 ? 'interrupted' : 'completed';
|
|
const started = new Date(
|
|
Date.UTC(2023, 0, 1) + Math.floor((id / definition.matches) * 3 * 365 * 86400000)
|
|
).toISOString();
|
|
const ended = new Date(
|
|
Date.parse(started) + (20 + Math.floor(random() * 130)) * 60000
|
|
).toISOString();
|
|
const round = id % 101 === 0 ? 1 : 2 + Math.floor(random() * 60);
|
|
match.run(
|
|
id,
|
|
`server_${id % 24}`,
|
|
game,
|
|
map,
|
|
`${game}:${map}`,
|
|
`Map ${mapIndex}`,
|
|
started,
|
|
status === 'completed' ? ended : null,
|
|
ended,
|
|
status,
|
|
round,
|
|
status === 'completed' ? 'game_end' : 'heartbeat',
|
|
`event_${id}`
|
|
);
|
|
const roster = new Set();
|
|
while (roster.size < 4) roster.add(Math.floor(random() ** 2 * definition.players));
|
|
// Reserve one completed participation for an explicitly ineligible player.
|
|
if (id === definition.matches - 2) roster.delete([...roster][3]);
|
|
for (const player of roster) {
|
|
const guid = String(10000000000000000n + BigInt(player));
|
|
const name = `${game === 't6' ? 'Alias' : 'Player'} ${String(player).padStart(5, '0')}`;
|
|
identity.run(game, guid, name, started, ended);
|
|
const kills = Math.floor(round * 18 + random() * 800);
|
|
const presentAtStart = random() > 0.1;
|
|
const firstRound = presentAtStart ? 0 : Math.max(1, Math.floor(random() * round));
|
|
participation.run(
|
|
id,
|
|
guid,
|
|
round,
|
|
Math.max(0, round - Math.max(1, firstRound)),
|
|
kills,
|
|
Math.floor(kills * random()),
|
|
Math.floor(random() * 8),
|
|
Math.floor(random() * 12),
|
|
Math.floor(random() * 100000),
|
|
Math.max(0, round - 1),
|
|
Number(status === 'active'),
|
|
Number(random() > 0.01),
|
|
Number(status === 'completed' && random() > 0.15),
|
|
started,
|
|
ended,
|
|
firstRound,
|
|
Number(presentAtStart)
|
|
);
|
|
}
|
|
if (id === definition.matches - 2) {
|
|
identity.run(game, 'benchmark-low-sample', 'Low sample', started, ended);
|
|
participation.run(
|
|
id,
|
|
'benchmark-low-sample',
|
|
2,
|
|
1,
|
|
20,
|
|
5,
|
|
0,
|
|
0,
|
|
1000,
|
|
1,
|
|
0,
|
|
1,
|
|
1,
|
|
started,
|
|
ended,
|
|
0,
|
|
1
|
|
);
|
|
}
|
|
}
|
|
})();
|
|
handle.pragma('wal_checkpoint(TRUNCATE)');
|
|
}
|
|
|
|
function loadImplementation(handle, ref) {
|
|
const modules = new Map();
|
|
const hashes = {};
|
|
let recorded = null;
|
|
const database = {
|
|
prepare(sql) {
|
|
const statement = handle.prepare(sql);
|
|
return Object.fromEntries(
|
|
['all', 'get'].map((method) => [
|
|
method,
|
|
(...parameters) => {
|
|
const start = performance.now();
|
|
const result = statement[method](...parameters);
|
|
if (recorded)
|
|
recorded.push({
|
|
sql,
|
|
parameters,
|
|
milliseconds: performance.now() - start
|
|
});
|
|
return result;
|
|
}
|
|
])
|
|
);
|
|
}
|
|
};
|
|
function load(file) {
|
|
if (file === 'src/lib/server/db/index.ts') return { db: () => database };
|
|
if (modules.has(file)) return modules.get(file);
|
|
const text = source(file, ref);
|
|
hashes[file] = createHash('sha256').update(text).digest('hex');
|
|
const compiled = ts.transpileModule(text, {
|
|
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }
|
|
}).outputText;
|
|
const exports = {};
|
|
modules.set(file, exports);
|
|
const localRequire = (specifier) => {
|
|
if (!specifier.startsWith('.')) return require(specifier);
|
|
const target = relative(repo, resolve(repo, dirname(file), specifier)).replace(
|
|
/\.js$/,
|
|
'.ts'
|
|
);
|
|
if (!target.startsWith('src/lib/'))
|
|
throw new Error(`Unexpected benchmark dependency: ${target}`);
|
|
return load(target);
|
|
};
|
|
new Function('require', 'exports', compiled)(localRequire, exports);
|
|
return exports;
|
|
}
|
|
const api = load('src/lib/server/fleet/leaderboard.ts');
|
|
return {
|
|
name: ref ? 'baseline' : 'current',
|
|
ref,
|
|
hashes,
|
|
metrics: load('src/lib/leaderboard.ts').leaderboardMetrics.map((metric) => metric.key),
|
|
run(test) {
|
|
return test.guid
|
|
? api.getPlayerLeaderboardEntry(test.guid, test.options)
|
|
: api.getLeaderboard(test.options);
|
|
},
|
|
measure(test) {
|
|
recorded = [];
|
|
const start = performance.now();
|
|
const result = this.run(test);
|
|
const milliseconds = performance.now() - start;
|
|
const statements = recorded;
|
|
recorded = null;
|
|
return { result, milliseconds, statements };
|
|
}
|
|
};
|
|
}
|
|
|
|
const report = {
|
|
environment: {
|
|
node: process.version,
|
|
sqlite: null,
|
|
platform: platform(),
|
|
arch: arch(),
|
|
cpu: cpus()[0]?.model
|
|
},
|
|
baselineRef: baseline,
|
|
runs: options.runs,
|
|
seed: 20260918,
|
|
notes: 'Synthetic file-backed SQLite, one warmup per case, rotating execution order, warm OS cache. Includes SQL preparation and source-record lookup; excludes HTTP and browser rendering. Shared local CPU; timings are not production latency. Schema and indexes are current for both implementations.',
|
|
scenarios: []
|
|
};
|
|
try {
|
|
for (const definition of definitions) {
|
|
const handle = new Database(join(temporary, `${definition.name}.sqlite`));
|
|
try {
|
|
report.environment.sqlite = handle
|
|
.prepare('SELECT sqlite_version() AS version')
|
|
.get().version;
|
|
seed(handle, definition);
|
|
const implementations = [
|
|
...(baseline ? [loadImplementation(handle, baseline)] : []),
|
|
loadImplementation(handle)
|
|
];
|
|
const cases = [
|
|
{ name: 'unfiltered', options: {} },
|
|
{ name: 'kills', options: { metric: 'kills' } },
|
|
{ name: 'game', options: { game: 't5', metric: 'kills' } },
|
|
{ name: 'map', options: { game: 't6', map: 't6:map_8', metric: 'kills' } },
|
|
{ name: 'search', options: { q: '00042', metric: 'kills' } },
|
|
{
|
|
name: 'deep-page',
|
|
options: { metric: 'kills', offset: Math.floor(definition.players / 2) }
|
|
},
|
|
{ name: 'efficiency', options: { metric: 'headshotRate' } },
|
|
{ name: 'exact-player', guid: '10000000000000042', options: { metric: 'kills' } },
|
|
{ name: 'empty-search', options: { q: 'No such player exists' } },
|
|
{ name: 'past-end', options: { offset: 999999 } },
|
|
{ name: 'empty-map', options: { map: 't5:missing' } }
|
|
];
|
|
const counts = handle
|
|
.prepare(
|
|
`SELECT COUNT(*) AS participationRows,
|
|
COUNT(DISTINCT guid) AS uniquePlayers FROM leaderboard_match_players`
|
|
)
|
|
.get();
|
|
const scenario = {
|
|
...definition,
|
|
...counts,
|
|
maps: 12,
|
|
servers: 24,
|
|
completedShare: 0.9,
|
|
results: []
|
|
};
|
|
report.scenarios.push(scenario);
|
|
// Broad parity is separate from timings so every ratio and edge case is
|
|
// checked without multiplying the larger fixture's timed request matrix.
|
|
if (baseline && definition.name === 'small') {
|
|
const scopes = [
|
|
{},
|
|
{ game: 't5' },
|
|
{ game: 't6', map: 't6:map_8' },
|
|
{ q: '00042' },
|
|
{ q: 'No such player exists' },
|
|
{ offset: 999999 },
|
|
{ map: 't5:missing' }
|
|
];
|
|
let checked = 0;
|
|
for (const metric of implementations[1].metrics) {
|
|
for (const scope of scopes) {
|
|
const test = { options: { ...scope, metric } };
|
|
assert.deepStrictEqual(
|
|
implementations[1].run(test),
|
|
implementations[0].run(test),
|
|
`${definition.name}/${metric}/${JSON.stringify(scope)}: full-response parity`
|
|
);
|
|
checked++;
|
|
}
|
|
for (const guid of [
|
|
'10000000000000042',
|
|
'benchmark-low-sample',
|
|
'benchmark-missing'
|
|
]) {
|
|
const test = { guid, options: { metric } };
|
|
assert.deepStrictEqual(
|
|
implementations[1].run(test),
|
|
implementations[0].run(test),
|
|
`${definition.name}/${metric}/${guid}: exact-player parity`
|
|
);
|
|
checked++;
|
|
}
|
|
}
|
|
scenario.parityCases = checked;
|
|
console.log(
|
|
JSON.stringify({
|
|
scenario: definition.name,
|
|
parityCases: checked,
|
|
parity: true
|
|
})
|
|
);
|
|
}
|
|
for (const test of cases) {
|
|
const expected = implementations[0].run(test);
|
|
for (const implementation of implementations.slice(1)) {
|
|
assert.deepStrictEqual(
|
|
implementation.run(test),
|
|
expected,
|
|
`${definition.name}/${test.name}: baseline parity`
|
|
);
|
|
}
|
|
const samples = Object.fromEntries(implementations.map((item) => [item.name, []]));
|
|
const latestStatements = {};
|
|
for (let index = 0; index < options.runs; index++) {
|
|
const rotation = index % implementations.length;
|
|
for (const implementation of [
|
|
...implementations.slice(rotation),
|
|
...implementations.slice(0, rotation)
|
|
]) {
|
|
const measured = implementation.measure(test);
|
|
samples[implementation.name].push(measured.milliseconds);
|
|
latestStatements[implementation.name] = measured.statements;
|
|
}
|
|
}
|
|
const result = {
|
|
...test,
|
|
parity: baseline ? true : null,
|
|
implementations: implementations.map((implementation) => ({
|
|
name: implementation.name,
|
|
sourceHashes: implementation.hashes,
|
|
medianMs: percentile(samples[implementation.name], 0.5),
|
|
p95Ms: percentile(samples[implementation.name], 0.95),
|
|
samplesMs: samples[implementation.name],
|
|
statements: latestStatements[implementation.name].map((statement) => ({
|
|
...statement,
|
|
plan: handle
|
|
.prepare(`EXPLAIN QUERY PLAN ${statement.sql}`)
|
|
.all(...statement.parameters)
|
|
}))
|
|
}))
|
|
};
|
|
scenario.results.push(result);
|
|
console.log(
|
|
JSON.stringify({
|
|
scenario: definition.name,
|
|
case: test.name,
|
|
parity: result.parity,
|
|
medianMs: Object.fromEntries(
|
|
result.implementations.map((item) => [
|
|
item.name,
|
|
Math.round(item.medianMs)
|
|
])
|
|
)
|
|
})
|
|
);
|
|
}
|
|
} finally {
|
|
handle.close();
|
|
}
|
|
}
|
|
if (options.json) {
|
|
const target = resolve(options.json);
|
|
mkdirSync(dirname(target), { recursive: true });
|
|
writeFileSync(target, JSON.stringify(report, null, 2));
|
|
console.log(`Detailed results: ${target}`);
|
|
}
|
|
} finally {
|
|
rmSync(temporary, { recursive: true, force: true });
|
|
}
|