Add native Discord boards and availability alerts
CI / checks (push) Failing after 7m41s

This commit is contained in:
2026-10-18 16:55:49 -04:00
parent f2b2541b46
commit fd00ef5357
29 changed files with 4685 additions and 3 deletions
+229
View File
@@ -0,0 +1,229 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { flushSync, mount, unmount } from 'svelte';
import { writable } from 'svelte/store';
import DiscordPage from '../../src/routes/settings/integrations/discord/+page.svelte';
import type { DiscordAdminState, DiscordSettings } from '../../src/lib/discord.js';
const page = writable({ data: { user: { role: 'admin' } } });
vi.mock('$app/stores', () => ({
page: { subscribe: (run: Parameters<typeof page.subscribe>[0]) => page.subscribe(run) }
}));
let component: ReturnType<typeof mount> | undefined;
beforeEach(() => {
vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('visible');
});
afterEach(async () => {
if (component) await unmount(component);
component = undefined;
document.body.replaceChildren();
vi.useRealTimers();
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
const serverName = 't5_zombie_theater_1';
const guildId = '12345678901234567';
const channelId = '23456789012345678';
function snapshot(): DiscordAdminState {
return {
settings: {
enabled: false,
guildId,
channels: { t5: channelId, t6: '' },
publishedServers: [],
showPlayers: false,
showAddresses: false,
leaderboardEnabled: false
},
credentials: {
applicationId: '34567890123456789',
publicKeyConfigured: true,
botTokenConfigured: true
},
status: {
lastAttemptAt: null,
lastSuccessAt: null,
lastError: null,
boards: [],
activeSubscriptions: 0,
pendingNotifications: 0
},
servers: [
{
name: serverName,
game: 't5',
label: 'Kino der Toten · Lobby 1',
map: 'Kino der Toten'
}
],
preview: { t5: [], t6: [] }
};
}
const text = (node: Element) => node.textContent?.replace(/\s+/g, ' ').trim();
const findButton = (label: string) =>
[...document.querySelectorAll('button')].find((node) => text(node) === label);
const button = (label: string) => {
const node = findButton(label);
expect(node, `Missing button: ${label}`).toBeDefined();
return node!;
};
const checkbox = (label: string) => {
const node = [...document.querySelectorAll('label')].find((node) =>
text(node)?.includes(label)
);
const input = node?.querySelector<HTMLInputElement>('input[type="checkbox"]');
expect(input, `Missing checkbox: ${label}`).toBeTruthy();
return input!;
};
const field = (label: string) => {
const node = [...document.querySelectorAll('label')].find((node) => text(node) === label)!;
const input = document.getElementById(node.htmlFor) as HTMLInputElement;
expect(input?.tagName).toBe('INPUT');
return input;
};
function enter(label: string, value: string) {
const input = field(label);
input.value = value;
input.dispatchEvent(new Event('input', { bubbles: true }));
flushSync();
}
function toggle(label: string) {
checkbox(label).click();
flushSync();
}
async function show(role = 'admin') {
page.set({ data: { user: { role } } });
component = mount(DiscordPage, { target: document.body });
flushSync();
await vi.waitFor(() => {
flushSync();
expect(document.querySelector('form')).not.toBeNull();
});
}
it('saves explicit publication choices without credentials and restores saved values on discard', async () => {
const initial = snapshot();
let submitted: DiscordSettings | undefined;
const fetch = vi.fn(async (_url: string, init?: RequestInit) => {
if (init?.method === 'PUT') {
submitted = JSON.parse(String(init.body)) as DiscordSettings;
return Response.json({ ...initial, settings: submitted });
}
return Response.json(initial);
});
vi.stubGlobal('fetch', fetch);
await show();
expect(checkbox('Enable Discord companion').checked).toBe(false);
expect(checkbox(serverName).checked).toBe(false);
expect(checkbox('Show player names').checked).toBe(false);
expect(checkbox('Show public server connection addresses').checked).toBe(false);
expect(checkbox('Publish fleet leaderboards').checked).toBe(false);
expect(document.body.textContent).toContain('including servers outside the selection above');
toggle('Enable Discord companion');
toggle(serverName);
toggle('Show player names');
toggle('Publish fleet leaderboards');
enter('Discord server ID', '45678901234567890');
expect(button('Register commands').disabled).toBe(true);
button('Save changes').click();
await vi.waitFor(() => expect(document.body.textContent).toContain('Settings saved.'));
expect(submitted).toEqual({
...initial.settings,
enabled: true,
guildId: '45678901234567890',
publishedServers: [serverName],
showPlayers: true,
leaderboardEnabled: true
});
expect(fetch.mock.calls.filter(([, init]) => init?.method === 'PUT')).toHaveLength(1);
expect(fetch.mock.calls.every(([url]) => url === '/api/integrations/discord')).toBe(true);
expect(button('Save changes').disabled).toBe(true);
toggle('Show player names');
toggle(serverName);
button('Discard').click();
flushSync();
expect(checkbox('Show player names').checked).toBe(true);
expect(checkbox(serverName).checked).toBe(true);
expect(button('Save changes').disabled).toBe(true);
});
it('keeps Super Viewer controls read-only and rejects a directly submitted mutation', async () => {
const fetch = vi.fn(async (_url: string, _init?: RequestInit) => Response.json(snapshot()));
vi.stubGlobal('fetch', fetch);
await show('super_viewer');
expect(document.body.textContent).toContain('You have read-only access.');
for (const group of document.querySelectorAll('fieldset')) expect(group.disabled).toBe(true);
expect(findButton('Save changes')).toBeUndefined();
expect(findButton('Register commands')).toBeUndefined();
document
.querySelector('form')!
.dispatchEvent(new Event('submit', { bubbles: true, cancelable: true }));
flushSync();
expect(fetch).toHaveBeenCalledTimes(1);
button('Refresh status').click();
await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(2));
expect(fetch.mock.calls.every((args) => !(args[1] as RequestInit | undefined)?.method)).toBe(
true
);
});
it('preserves publication edits and the last snapshot across polling failures and recovery', async () => {
vi.useFakeTimers();
const initial = snapshot();
const updated = snapshot();
updated.status.activeSubscriptions = 3;
const fetch = vi
.fn()
.mockResolvedValueOnce(Response.json(initial))
.mockRejectedValueOnce(new Error('Discord status unavailable.'))
.mockResolvedValueOnce(Response.json(updated));
vi.stubGlobal('fetch', fetch);
await show();
toggle(serverName);
enter('Discord server ID', '45678901234567890');
await vi.advanceTimersByTimeAsync(30_000);
flushSync();
expect(document.querySelector('[role="alert"]')?.textContent).toContain(
'Discord status unavailable.'
);
expect(document.body.textContent).toContain('Kino der Toten · Lobby 1');
expect(checkbox(serverName).checked).toBe(true);
expect(field('Discord server ID').value).toBe('45678901234567890');
button('Retry loading').click();
await vi.waitFor(() => expect(document.querySelector('[role="alert"]')).toBeNull());
expect(checkbox(serverName).checked).toBe(true);
expect(field('Discord server ID').value).toBe('45678901234567890');
expect(button('Save changes').disabled).toBe(false);
expect(document.body.textContent).toContain('Unsaved changes');
});
it('registers commands only on an explicit action and retains its error through successful refreshes', async () => {
const fetch = vi.fn(async (url: string) =>
url.endsWith('/register')
? Response.json({ error: 'Command registration denied.' }, { status: 502 })
: Response.json(snapshot())
);
vi.stubGlobal('fetch', fetch);
await show();
expect(fetch.mock.calls.some(([url]) => url.endsWith('/register'))).toBe(false);
button('Register commands').click();
await vi.waitFor(() =>
expect(document.body.textContent).toContain('Command registration denied.')
);
await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(3));
expect(fetch.mock.calls[1]).toEqual(['/api/integrations/discord/register', { method: 'POST' }]);
button('Refresh status').click();
await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(4));
expect(document.querySelector('[role="alert"]')?.textContent).toContain(
'Command registration denied.'
);
});
+417
View File
@@ -0,0 +1,417 @@
import Database from 'better-sqlite3';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
defaultDiscordSettings,
type DiscordMessage,
type DiscordPublicServer
} from '../../src/lib/discord.js';
import { migrateDiscord } from '../../src/lib/server/discord/schema.js';
import { DiscordStore } from '../../src/lib/server/discord/store.js';
import {
DiscordDeliveryError,
DiscordRest,
type DiscordTransport
} from '../../src/lib/server/discord/transport.js';
import { DiscordWorker } from '../../src/lib/server/discord/worker.js';
const guildId = '12345678901234567';
const channelId = '23456789012345678';
const userId = '34567890123456789';
const messageId = '45678901234567890';
const dmId = '56789012345678901';
let handle: Database.Database;
let store: DiscordStore;
let now: number;
let server: DiscordPublicServer;
const message = (content: string): DiscordMessage => ({ content, allowed_mentions: { parse: [] } });
const watchInput = {
guildId,
userId,
game: 't5' as const,
mapKey: 't5:zombie_theater',
mode: 'empty' as const
};
beforeEach(() => {
handle = new Database(':memory:');
migrateDiscord(handle);
now = Date.parse('2026-10-18T20:00:00Z');
store = new DiscordStore(handle, () => now);
store.saveSettings({
...defaultDiscordSettings,
enabled: true,
guildId,
channels: { t5: channelId, t6: '' },
publishedServers: ['kino']
});
server = {
name: 'kino',
game: 't5',
label: 'Kino lobby',
map: 'Kino der Toten',
mapKey: watchInput.mapKey,
state: 'empty',
players: 0,
capacity: 4,
round: 1,
observedAt: new Date(now).toISOString(),
playerNames: [],
connection: null
};
});
afterEach(() => {
handle.close();
vi.restoreAllMocks();
});
function worker(rest: DiscordTransport, render = () => [message('Public board')]) {
return new DiscordWorker({
store,
transport: () => rest,
fleet: (settings) => (settings.publishedServers.includes(server.name) ? [server] : []),
render,
now: () => now
});
}
function transport(handler?: (method: string, path: string, body?: unknown) => unknown) {
const request = vi.fn(async (method: string, path: string, body?: unknown) => {
if (handler) {
const result = await handler(method, path, body);
if (result !== undefined) return result;
}
if (path === `/channels/${channelId}`) return { guild_id: guildId, type: 0 };
if (path === '/users/@me/channels') return { id: dmId };
return { id: messageId };
});
return { request, rest: { request } as DiscordTransport };
}
function advance() {
now += 30_000;
server.observedAt = new Date(now).toISOString();
}
describe('Discord delivery storage', () => {
it('retains interaction replay claims after cancellation and database reopen, then prunes expiry', () => {
const interactionId = '67890123456789012';
expect(store.claimInteraction(interactionId)).toBe(true);
const watch = store.subscribe(watchInput);
store.cancel(watch.id, guildId, userId);
const serialized = handle.serialize();
handle.close();
handle = new Database(serialized);
store = new DiscordStore(handle, () => now);
expect(store.claimInteraction(interactionId)).toBe(false);
expect(store.subscriptions(userId, guildId)).toHaveLength(0);
now += 15 * 60_000;
expect(store.claimInteraction(interactionId)).toBe(true);
expect(() => store.claimInteraction('not-a-snowflake')).toThrow(
'Invalid Discord interaction'
);
});
it('replays the migration and persists settings, board IDs, and opt-in watches', () => {
const watch = store.subscribe(watchInput);
store.reconcileBoards([{ game: 't5', guildId, channelId, messages: [message('Board')] }]);
store.delivered(store.jobs()[0], messageId);
const serialized = handle.serialize();
handle.close();
handle = new Database(serialized);
const schemaVersion = handle.pragma('schema_version', { simple: true });
migrateDiscord(handle);
expect(handle.pragma('schema_version', { simple: true })).toBe(schemaVersion);
store = new DiscordStore(handle, () => now);
expect(store.settings().guildId).toBe(guildId);
expect(store.boards()[0].messageId).toBe(messageId);
expect(store.subscriptions(userId, guildId)[0].id).toBe(watch.id);
});
it('deduplicates watches, bounds each member, and scopes cancellation to the owner and guild', () => {
const watch = store.subscribe(watchInput);
expect(store.subscribe(watchInput)).toEqual(watch);
for (let i = 1; i < 10; i++) store.subscribe({ ...watchInput, mapKey: `t5:map_${i}` });
expect(() => store.subscribe({ ...watchInput, mapKey: 't5:eleventh' })).toThrow(
'10 active'
);
expect(store.cancel(watch.id, guildId, '99999999999999999')).toBe(false);
expect(store.cancel(watch.id, '99999999999999999', userId)).toBe(false);
expect(store.cancel(watch.id, guildId, userId)).toBe(true);
expect(store.subscriptions(userId, guildId)).toHaveLength(9);
});
it('expires watches and their pending notifications, and bounds durable work atomically', () => {
const watch = store.subscribe(watchInput);
store.queueAlert(store.subscription(watch.id)!, message('Alert'));
expect(store.jobs()).toHaveLength(1);
now += 24 * 60 * 60_000;
store.expireSubscriptions();
expect(store.subscription(watch.id)?.state).toBe('expired');
expect(store.jobs()).toHaveLength(0);
expect(() =>
store.reconcileBoards([
{
game: 't5',
guildId,
channelId,
messages: Array.from({ length: 501 }, (_, i) => message(String(i)))
}
])
).toThrow('queue is full');
expect(store.boards()).toHaveLength(0);
});
});
describe('Discord board worker', () => {
it('retires the original guild board before rejecting a replacement in the wrong guild', async () => {
const { rest, request } = transport();
const instance = worker(rest);
await instance.tick();
const original = store.boards()[0];
expect(original.guildId).toBe(guildId);
store.saveSettings({ ...store.settings(), guildId: '99999999999999999' });
// The message retains its original guild until the deletion is reconciled.
expect(store.board(original.id)?.guildId).toBe(guildId);
request.mockClear();
await instance.tick();
expect(request.mock.calls.map(([method]) => method)).toEqual(['DELETE', 'GET']);
expect(request.mock.calls[0][1]).toBe(`/channels/${channelId}/messages/${messageId}`);
expect(store.board(original.id)).toBeUndefined();
expect(store.boards()).toEqual([
expect.objectContaining({ guildId: '99999999999999999', messageId: null })
]);
expect(store.status().lastError).toContain('does not belong');
});
it('creates split boards once, survives a worker restart, and skips unchanged edits', async () => {
const { rest, request } = transport();
const render = () => [message('Page 1'), message('Page 2')];
await worker(rest, render).tick();
expect(request.mock.calls.filter(([method]) => method === 'POST')).toHaveLength(2);
expect(store.boards()).toHaveLength(2);
request.mockClear();
await worker(rest, render).tick();
expect(request).not.toHaveBeenCalled();
await worker(rest, () => [message('Changed page')]).tick();
expect(request.mock.calls.some(([method]) => method === 'PATCH')).toBe(true);
expect(request.mock.calls.some(([method]) => method === 'DELETE')).toBe(true);
expect(store.boards()).toHaveLength(1);
});
it('recreates deleted messages with a new nonce while preserving other board pages', async () => {
let missing = false;
const { rest, request } = transport((method, path) => {
if (missing && method === 'PATCH' && path.includes('/messages/'))
throw new DiscordDeliveryError('Missing message.', 404, null, 10008);
});
await worker(rest).tick();
const firstNonce = (
request.mock.calls.find(([method]) => method === 'POST')![2] as { nonce: string }
).nonce;
missing = true;
request.mockClear();
await worker(rest, () => [message('Updated')]).tick();
const secondNonce = (
request.mock.calls.find(([method]) => method === 'POST')![2] as { nonce: string }
).nonce;
expect(secondNonce).not.toBe(firstNonce);
expect(store.boards()[0].messageId).toBe(messageId);
expect(store.jobs()).toHaveLength(0);
});
it('detects unchanged messages deleted externally during periodic verification', async () => {
let missing = false;
const { rest, request } = transport((method, path) => {
if (missing && method === 'GET' && path.includes('/messages/'))
throw new DiscordDeliveryError('Missing message.', 404, null, 10008);
});
const instance = worker(rest);
await instance.tick();
now += 16 * 60_000;
missing = true;
await instance.tick();
expect(store.boards()[0].messageId).toBeNull();
request.mockClear();
await instance.tick();
expect(request.mock.calls.some(([method]) => method === 'POST')).toBe(true);
});
it('refuses channels outside the allowed guild and pauses permission failures until retry', async () => {
const { rest, request } = transport((method, path) =>
method === 'GET' && path === `/channels/${channelId}`
? { guild_id: '99999999999999999', type: 0 }
: undefined
);
const instance = worker(rest);
await instance.tick();
expect(request.mock.calls.some(([method]) => method === 'POST')).toBe(false);
expect(store.status().lastError).toContain('does not belong');
request.mockClear();
await instance.tick();
expect(request).not.toHaveBeenCalled();
store.retryFailed();
await instance.tick();
expect(request).toHaveBeenCalledOnce();
});
it('checks settings again after channel lookup and removes board data when disabled', async () => {
const { rest, request } = transport();
const instance = worker(rest);
await instance.tick();
store.saveSettings({ ...store.settings(), enabled: false });
request.mockClear();
await instance.tick();
expect(request.mock.calls.map(([method]) => method)).toEqual(['DELETE']);
expect(store.boards()).toHaveLength(0);
store.saveSettings({ ...store.settings(), enabled: true });
const disabling = transport((method) => {
if (method === 'GET') store.saveSettings({ ...store.settings(), enabled: false });
});
await worker(disabling.rest).tick();
expect(disabling.request.mock.calls.some(([method]) => method === 'POST')).toBe(false);
});
it('persists rate-limit cooldowns without blocking the tick or creating overlapping runs', async () => {
let limited = true;
const { rest, request } = transport((method) => {
if (method === 'POST' && limited)
throw new DiscordDeliveryError('Rate limited.', 429, now + 120_000);
});
const instance = worker(rest);
expect(instance.tick()).toBe(instance.tick());
await instance.tick();
request.mockClear();
advance();
await instance.tick();
expect(request).not.toHaveBeenCalled();
now += 120_000;
limited = false;
await instance.tick();
expect(store.boards()[0].messageId).toBe(messageId);
});
});
describe('Discord one-time availability notifications', () => {
it('requires stable fresh observations and delivers only one private DM without mentions', async () => {
const watch = store.subscribe(watchInput);
const { rest, request } = transport();
const instance = worker(rest, () => []);
await instance.tick();
expect(request).not.toHaveBeenCalled();
advance();
await instance.tick();
expect(request.mock.calls.map(([, path]) => path)).toEqual([
'/users/@me/channels',
`/channels/${dmId}/messages`
]);
expect(request.mock.calls[1][2]).toMatchObject({
allowed_mentions: { parse: [] },
enforce_nonce: true
});
expect(store.subscription(watch.id)?.state).toBe('delivered');
request.mockClear();
advance();
await instance.tick();
expect(request).not.toHaveBeenCalled();
});
it('does not notify on stale data or unknown capacity for a space watch', async () => {
store.subscribe({ ...watchInput, mode: 'space' });
const { rest, request } = transport();
const instance = worker(rest, () => []);
server.capacity = null;
await instance.tick();
advance();
await instance.tick();
expect(request).not.toHaveBeenCalled();
server.capacity = 4;
server.state = 'unknown';
advance();
await instance.tick();
expect(request).not.toHaveBeenCalled();
server.state = 'empty';
server.observedAt = new Date(now - 100_000).toISOString();
await instance.tick();
expect(request).not.toHaveBeenCalled();
});
it('withdraws queued alerts if a server is unpublished while the DM channel opens', async () => {
const watch = store.subscribe(watchInput);
const { rest, request } = transport((method, path) => {
if (path === '/users/@me/channels')
store.saveSettings({ ...store.settings(), publishedServers: [] });
});
const instance = worker(rest, () => []);
await instance.tick();
advance();
await instance.tick();
expect(request.mock.calls).toHaveLength(1);
expect(store.subscription(watch.id)?.state).toBe('active');
expect(store.jobs()).toHaveLength(0);
});
it('stops retrying forbidden DMs and exposes the failure in delivery health', async () => {
const watch = store.subscribe(watchInput);
const { rest, request } = transport((method, path) => {
if (path.includes('/messages'))
throw new DiscordDeliveryError('Recipient does not accept direct messages.', 403);
});
const instance = worker(rest, () => []);
await instance.tick();
advance();
await instance.tick();
expect(store.subscription(watch.id)?.state).toBe('failed');
expect(store.status().lastError).toContain('direct messages');
request.mockClear();
advance();
await instance.tick();
expect(request).not.toHaveBeenCalled();
});
});
describe('Discord REST transport', () => {
it('honors global retry_after and route reset headers without network calls during cooldown', async () => {
const fetcher = vi.fn(
async () =>
new Response(JSON.stringify({ retry_after: 12.5, global: true }), { status: 429 })
);
const rest = new DiscordRest('private-token', fetcher, () => now);
await expect(
rest.request('POST', `/channels/${channelId}/messages`, {})
).rejects.toMatchObject({ status: 429, retryAt: now + 12_500 });
await expect(rest.request('GET', `/channels/${dmId}`)).rejects.toMatchObject({
status: 429
});
expect(fetcher).toHaveBeenCalledOnce();
now += 13_000;
fetcher.mockImplementation(
async () =>
new Response('{"id":"123"}', {
headers: {
'X-RateLimit-Remaining': '0',
'X-RateLimit-Reset-After': '3',
'X-RateLimit-Bucket': 'messages'
}
})
);
await rest.request('POST', `/channels/${channelId}/messages`, {});
await expect(
rest.request('POST', `/channels/${channelId}/messages`, {})
).rejects.toMatchObject({ retryAt: now + 3000 });
expect(fetcher).toHaveBeenCalledTimes(2);
});
it('never exposes response bodies or tokens and stops HTTP requests after invalid credentials', async () => {
const fetcher = vi.fn(
async () => new Response('{"message":"private response data"}', { status: 401 })
);
const rest = new DiscordRest('private-token', fetcher, () => now);
await expect(rest.request('GET', `/channels/${channelId}`)).rejects.toThrow(
'credentials were rejected'
);
await expect(rest.request('GET', `/channels/${channelId}`)).rejects.not.toThrow('private');
expect(fetcher).toHaveBeenCalledOnce();
await expect(rest.request('GET', 'https://attacker.invalid')).rejects.toThrow(
'Invalid Discord API path'
);
expect(fetcher).toHaveBeenCalledOnce();
});
});
+471
View File
@@ -0,0 +1,471 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
defaultDiscordSettings,
type DiscordMessage,
type DiscordSettings
} from '../../src/lib/discord.js';
import type { FleetServer } from '../../src/lib/fleet-overview.js';
import type { LeaderboardResponse } from '../../src/lib/leaderboard.js';
import { gameEvent } from '../helpers/telemetry.js';
vi.mock('../../src/lib/server/fleet/snapshot.js', () => ({ fleetSnapshot: vi.fn() }));
vi.mock('../../src/lib/server/reports/index.js', () => ({ leaderboardReport: vi.fn() }));
vi.mock('../../src/lib/server/discord/store.js', () => ({
getDiscordSettings: vi.fn(),
subscribeToMap: vi.fn(),
listSubscriptions: vi.fn(),
cancelSubscription: vi.fn()
}));
import { fleetSnapshot } from '../../src/lib/server/fleet/snapshot.js';
import { leaderboardReport } from '../../src/lib/server/reports/index.js';
import {
getDiscordSettings,
subscribeToMap,
listSubscriptions,
cancelSubscription
} from '../../src/lib/server/discord/store.js';
import {
discordServerKey,
discordText,
projectFleet
} from '../../src/lib/server/discord/projection.js';
import {
browserId,
filterServers,
renderBoard,
renderBrowser,
renderServer
} from '../../src/lib/server/discord/render.js';
import {
handleDiscordInteraction,
type DiscordInteraction
} from '../../src/lib/server/discord/interactions.js';
import { discordCommands, discordMaps } from '../../src/lib/server/discord/commands.js';
const now = Date.parse('2026-10-18T20:00:00Z');
const stamp = new Date(now - 10000).toISOString();
const guildId = '111111111111111111';
const userId = '222222222222222222';
const applicationId = '333333333333333333';
const name = 't5_zombie_theater_1';
let settings: DiscordSettings;
function server(overrides: Partial<FleetServer> = {}): FleetServer {
return {
name,
game: 't5',
map: 'zombie_theater',
map_slug: 'theater',
game_mode: 'zombie',
health: 'online',
last_players: 2,
capacity: 4,
capacity_observed_at: stamp,
last_map: 'zombie_theater',
last_gametype: 'zombie',
last_seen_at: stamp,
last_error: null,
held: [],
active_since: new Date(now - 60000).toISOString(),
inactive_since: null,
game_started_at: new Date(now - 60000).toISOString(),
game_ended_at: null,
roster: [
{
name: '@everyone <@123> **spoof**',
guid: 'private-guid',
score: 1000,
num: 0,
ping: 10
}
],
telemetry: { ...gameEvent('heartbeat', 28, 2), receivedAt: stamp },
...overrides
};
}
function interaction(data: DiscordInteraction['data'], type = 2): DiscordInteraction {
return {
type,
application_id: applicationId,
guild_id: guildId,
member: { user: { id: userId } },
data
};
}
function components(message: DiscordMessage): Record<string, unknown>[] {
const result: Record<string, unknown>[] = [];
const visit = (component: Record<string, unknown>) => {
result.push(component);
for (const child of (component.components ?? []) as Record<string, unknown>[]) visit(child);
if (component.accessory) visit(component.accessory as Record<string, unknown>);
};
for (const component of message.components ?? []) visit(component);
return result;
}
function messageText(message: DiscordMessage): string {
return components(message)
.map((component) => component.content ?? '')
.join('\n');
}
beforeEach(() => {
vi.clearAllMocks();
vi.useFakeTimers();
vi.setSystemTime(now);
vi.stubEnv('EC_DISCORD_APPLICATION_ID', applicationId);
settings = {
...structuredClone(defaultDiscordSettings),
enabled: true,
guildId,
channels: { t5: '444444444444444444', t6: '' },
publishedServers: [name]
};
vi.mocked(getDiscordSettings).mockImplementation(() => settings);
vi.mocked(fleetSnapshot).mockReturnValue({ servers: [server()] } as ReturnType<
typeof fleetSnapshot
>);
vi.mocked(subscribeToMap).mockReturnValue({ id: 'watch-123', expiresAt: now + 86400000 });
vi.mocked(listSubscriptions).mockReturnValue([]);
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
});
describe('Discord publication boundary', () => {
it('projects only opted-in servers and explicitly approved fields', () => {
const raw = {
...server(),
host: { id: 'internal-host' },
endpoint: { host: 'secret-rcon', port: 28960 },
network: { publicAddress: 'games.example.com', publicPort: 28960 },
secret: 'token-value'
};
const projected = projectFleet(
[raw as unknown as FleetServer, server({ name: 'unpublished' })],
settings,
now
);
expect(projected).toHaveLength(1);
expect(projected[0]).toMatchObject({
name,
state: 'in-progress',
players: 2,
capacity: 4,
round: 28,
mapKey: 't5:zombie_theater',
playerNames: [],
connection: null
});
for (const secret of [
'internal-host',
'secret-rcon',
'private-guid',
'token-value',
'@everyone'
])
expect(JSON.stringify(projected)).not.toContain(secret);
const optedIn = projectFleet(
[raw as unknown as FleetServer],
{ ...settings, showPlayers: true, showAddresses: true },
now
)[0];
expect(optedIn.playerNames).toHaveLength(1);
expect(optedIn.connection).toBe('games.example.com:28960');
expect(JSON.stringify(optedIn)).not.toContain('private-guid');
expect(
projectFleet([server()], { ...settings, showAddresses: true }, now)[0].connection
).toBeNull();
});
it.each([
[{ last_players: 0 }, 'empty', 0],
[{ last_players: 4 }, 'full', 4],
[{ last_players: null }, 'unknown', null],
[{ health: 'starting' }, 'starting', null],
[{ health: 'offline' }, 'offline', null],
[{ health: 'stale' }, 'stale', null],
[{ last_seen_at: new Date(now - 100000).toISOString() }, 'stale', null],
[{ last_error: 'timeout: secret diagnostics' }, 'stale', null]
] as const)('reports truthful state for %j', (change, state, players) => {
const projected = projectFleet([server(change)], settings, now)[0];
expect(projected).toMatchObject({ state, players });
if (players === null)
expect(projected).toMatchObject({ capacity: null, round: null, playerNames: [] });
});
it('never treats stale or previous-process capacity as an available space', () => {
for (const change of [
{ capacity_observed_at: new Date(now - 100000).toISOString() },
{ telemetry_after: new Date(now - 5000).toISOString() }
]) {
const projected = projectFleet([server(change)], settings, now);
expect(projected[0].capacity).toBeNull();
expect(filterServers(projected, { status: 'space' })).toEqual([]);
}
});
it('preserves T6 variants and drops registered variants after a live map change', () => {
const town = server({
name: 't6_zombie_town_1',
game: 't6',
map: 'zm_transit',
last_map: 'zm_transit',
map_slug: 'town',
telemetry: undefined
});
const setup = { ...settings, publishedServers: [town.name] };
expect(projectFleet([town], setup, now)[0]).toMatchObject({
map: 'Town Survival',
mapKey: 't6:zm_transit:town'
});
expect(projectFleet([{ ...town, last_map: 'zm_tomb' }], setup, now)[0]).toMatchObject({
map: 'Origins',
mapKey: 't6:zm_tomb:tomb'
});
});
it('neutralizes mentions, markdown links and control characters in rendered player names', () => {
const projected = projectFleet([server()], { ...settings, showPlayers: true }, now)[0];
const message = renderServer(projected);
expect(message.allowed_mentions).toEqual({ parse: [] });
expect(messageText(message)).not.toContain('@everyone');
expect(messageText(message)).not.toContain('<@123>');
expect(discordText('[link](https://example.com)\n@here')).toBe(
'\\[link\\]\\(https://example.com\\) @\u200bhere'
);
expect(JSON.stringify(message)).not.toContain('private-guid');
});
});
describe('native Discord messages', () => {
it('paginates large boards within Discord limits with unique component IDs', () => {
const fleet = Array.from({ length: 100 }, (_, i) => ({
...projectFleet([server()], settings, now)[0],
name: `server-${i}`,
label: '*'.repeat(128),
map: '_'.repeat(255),
players: Number.MAX_SAFE_INTEGER,
capacity: Number.MAX_SAFE_INTEGER,
round: Number.MAX_SAFE_INTEGER
}));
const boards = renderBoard('t5', fleet);
expect(boards).toHaveLength(13);
for (const message of [
...boards,
renderBrowser(fleet),
renderBrowser(fleet, { page: 1 })
]) {
const all = components(message);
expect(all.length).toBeLessThanOrEqual(40);
expect(messageText(message).length).toBeLessThanOrEqual(4000);
const ids = all.flatMap((component) =>
typeof component.custom_id === 'string' ? [component.custom_id] : []
);
expect(new Set(ids).size).toBe(ids.length);
expect(ids.every((id) => id.length <= 100)).toBe(true);
expect(message).not.toHaveProperty('content');
expect(message).not.toHaveProperty('embeds');
}
expect(boards.every((message) => message.flags === 32768)).toBe(true);
expect(renderBrowser(fleet).flags).toBe(32832);
});
it('defines only supported guild commands with bounded choices and required options first', () => {
expect(discordCommands.map((command) => command.name)).toEqual([
'servers',
'server',
'leaderboard',
'watch',
'help'
]);
expect(JSON.stringify(discordCommands)).not.toContain('autocomplete');
interface Schema {
name: string;
description: string;
required?: boolean;
options?: Schema[];
choices?: { name: string; value: string | number }[];
}
const inspect = (entries: Schema[]) => {
expect(entries.length).toBeLessThanOrEqual(25);
expect(new Set(entries.map((entry) => entry.name)).size).toBe(entries.length);
let optional = false;
for (const entry of entries) {
expect(entry.name).toMatch(/^[a-z_-]{1,32}$/);
expect(entry.description.length).toBeLessThanOrEqual(100);
if (entry.required) expect(optional).toBe(false);
else optional = true;
if (entry.options) inspect(entry.options);
if (entry.choices) {
expect(entry.choices.length).toBeLessThanOrEqual(25);
for (const choice of entry.choices) {
expect(choice.name.length).toBeLessThanOrEqual(100);
expect(String(choice.value).length).toBeLessThanOrEqual(100);
}
}
}
};
inspect(discordCommands);
for (const map of discordMaps) {
for (const status of ['all', 'empty', 'space', 'full', 'in-progress'] as const) {
expect(
`${browserId({ map: map.value, status, page: 100000 })}:next`.length
).toBeLessThanOrEqual(100);
}
}
expect(
discordCommands.find((command) => command.name === 'watch')?.options?.[0]
).toMatchObject({
name: 'add',
options: [
expect.objectContaining({ name: 'game', required: true }),
expect.objectContaining({ name: 'map', required: true }),
expect.objectContaining({ name: 'condition', required: true })
]
});
});
});
describe('Discord interaction isolation', () => {
it.each(['guild', 'user', 'application', 'disabled'] as const)(
'rejects unauthorized %s before reading fleet or reports',
async (failure) => {
const input = interaction({ name: 'servers' });
if (failure === 'guild') input.guild_id = '555555555555555555';
if (failure === 'user') input.member = {};
if (failure === 'application') input.application_id = '555555555555555555';
if (failure === 'disabled') settings.enabled = false;
const result = await handleDiscordInteraction(input);
expect(result.type).toBe(4);
expect(result.data.flags).toBe(32832);
expect(fleetSnapshot).not.toHaveBeenCalled();
expect(leaderboardReport).not.toHaveBeenCalled();
}
);
it('opens board details privately and refuses old unpublished IDs', async () => {
const result = await handleDiscordInteraction(
interaction({ custom_id: `ec:d:${discordServerKey(name)}` }, 3)
);
expect(result.type).toBe(4);
expect(result.data.flags).toBe(32832);
expect(messageText(result.data)).toContain('Round 28');
settings.publishedServers = [];
expect(
messageText(
(
await handleDiscordInteraction(
interaction({ custom_id: `ec:d:${discordServerKey(name)}` }, 3)
)
).data
)
).toContain('no longer published');
});
it('keeps filtered pagination private and rejects forged filters', async () => {
const result = await handleDiscordInteraction(
interaction(
{
custom_id: browserId({
game: 't5',
map: 't5:zombie_theater',
status: 'empty',
page: 900
})
},
3
)
);
expect(result.data.flags).toBe(32832);
expect(messageText(result.data)).toContain('0 matching servers');
const bad = await handleDiscordInteraction(interaction({ custom_id: 'ec:b:t7:all:0:' }, 3));
expect(messageText(bad.data)).toContain('invalid');
});
it('creates one-shot watches only for published maps and scopes cancellation to the actor', async () => {
const result = await handleDiscordInteraction(
interaction({ custom_id: `ec:w:empty:${discordServerKey('t5:zombie_theater')}` }, 3)
);
expect(subscribeToMap).toHaveBeenCalledWith({
guildId,
userId,
game: 't5',
mapKey: 't5:zombie_theater',
mode: 'empty'
});
expect(messageText(result.data)).toContain('one DM');
vi.mocked(subscribeToMap).mockClear();
settings.publishedServers = [];
await handleDiscordInteraction(
interaction({ custom_id: `ec:w:space:${discordServerKey('t5:zombie_theater')}` }, 3)
);
expect(subscribeToMap).not.toHaveBeenCalled();
await handleDiscordInteraction(
interaction({
name: 'watch',
options: [{ name: 'cancel', options: [{ name: 'id', value: 'other-watch' }] }]
})
);
expect(cancelSubscription).toHaveBeenCalledWith('other-watch', guildId, userId);
});
it('requires explicit leaderboard publication and reuses the qualified report query without leaking identities', async () => {
const input = interaction({
name: 'leaderboard',
options: [
{ name: 'game', value: 't5' },
{ name: 'map', value: 't5:zombie_theater' },
{ name: 'metric', value: 'highestRound' },
{ name: 'squad', value: 2 },
{ name: 'page', value: 3 }
]
});
await handleDiscordInteraction(input);
expect(leaderboardReport).not.toHaveBeenCalled();
settings.leaderboardEnabled = true;
vi.mocked(leaderboardReport).mockResolvedValue({
metric: 'highestRound',
offset: 20,
total: 21,
limit: 10,
maps: [{ key: 't5:zombie_theater', name: 'Kino Der Toten' }],
entries: [
{
rank: 21,
name: '@everyone',
guid: 'private-guid',
highestRound: 30,
records: { server: 'private-server' }
}
]
} as unknown as LeaderboardResponse);
const result = await handleDiscordInteraction(input);
expect(leaderboardReport).toHaveBeenCalledWith({
game: 't5',
map: 't5:zombie_theater',
metric: 'highestRound',
players: 2,
offset: 20,
limit: 10
});
expect(messageText(result.data)).toContain('Completed matches');
expect(messageText(result.data)).not.toContain('@everyone');
expect(JSON.stringify(result)).not.toContain('private-guid');
expect(JSON.stringify(result)).not.toContain('private-server');
});
it('drops an in-flight leaderboard when publication is disabled', async () => {
settings.leaderboardEnabled = true;
vi.mocked(leaderboardReport).mockImplementation(async () => {
settings.leaderboardEnabled = false;
return {} as LeaderboardResponse;
});
const result = await handleDiscordInteraction(interaction({ name: 'leaderboard' }));
expect(messageText(result.data)).toContain('no longer enabled');
});
});
+250
View File
@@ -0,0 +1,250 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { generateKeyPairSync, sign } from 'node:crypto';
import { defaultDiscordSettings, validateDiscordSettings } from '../../src/lib/discord.js';
import { routePermission, roleAllows } from '../../src/lib/server/auth/access.js';
import {
completeDiscordInteraction,
validInteractionEnvelope,
verifiedDiscordBody
} from '../../src/lib/server/discord/webhook.js';
const { privateKey, publicKey } = generateKeyPairSync('ed25519');
const applicationId = '123456789012345678';
const guildId = '223456789012345678';
const userId = '323456789012345678';
const settings = vi.hoisted(() => ({ enabled: true, guildId: '223456789012345678' }));
const handle = vi.hoisted(() => vi.fn());
vi.mock('../../src/lib/server/discord/store.js', () => ({
getDiscordSettings: () => settings,
claimDiscordInteraction: () => true
}));
vi.mock('../../src/lib/server/discord/interactions.js', () => ({
handleDiscordInteraction: handle
}));
function signedRequest(body: unknown, timestamp = String(Math.floor(Date.now() / 1000))) {
const raw = typeof body === 'string' ? body : JSON.stringify(body);
const signature = sign(null, Buffer.from(timestamp + raw), privateKey).toString('hex');
return new Request('https://ec.example/api/discord/interactions', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Signature-Ed25519': signature,
'X-Signature-Timestamp': timestamp
},
body: raw
});
}
const interaction = () => ({
id: '423456789012345678',
application_id: applicationId,
guild_id: guildId,
token: 'private-interaction-token',
type: 2,
member: { user: { id: userId } },
data: { name: 'servers' }
});
beforeEach(() => {
vi.resetModules();
vi.stubEnv(
'EC_DISCORD_PUBLIC_KEY',
publicKey.export({ type: 'spki', format: 'der' }).subarray(-32).toString('hex')
);
vi.stubEnv('EC_DISCORD_APPLICATION_ID', applicationId);
settings.enabled = true;
handle.mockReset().mockResolvedValue({
type: 4,
data: { flags: 64, content: 'Private results', allowed_mentions: { parse: [] } }
});
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
vi.unstubAllGlobals();
});
describe('Discord signed webhook boundary', () => {
it('verifies original bytes and accepts signed PING before enabling a bot', async () => {
expect(await verifiedDiscordBody(signedRequest({ type: 1 }))).toEqual({ type: 1 });
settings.enabled = false;
const { POST } = await import('../../src/routes/api/discord/interactions/+server.js');
expect(await (await POST({ request: signedRequest({ type: 1 }) })).json()).toEqual({
type: 1
});
expect(handle).not.toHaveBeenCalled();
});
it('rejects tampering, missing signatures and timestamps outside the replay window', async () => {
const original = signedRequest({ type: 1 });
const altered = new Request(original.url, {
method: 'POST',
headers: original.headers,
body: '{"type":2}'
});
await expect(verifiedDiscordBody(altered)).rejects.toMatchObject({ status: 401 });
await expect(
verifiedDiscordBody(new Request(original.url, { method: 'POST', body: '{}' }))
).rejects.toMatchObject({ status: 401 });
for (const offset of [-360_000, 360_000])
await expect(
verifiedDiscordBody(
signedRequest({ type: 1 }, String(Math.floor((Date.now() + offset) / 1000)))
)
).rejects.toMatchObject({ status: 401 });
});
it('bounds signed request bodies and rejects malformed objects', async () => {
await expect(
verifiedDiscordBody(signedRequest('a'.repeat(65 * 1024)))
).rejects.toMatchObject({ status: 413 });
await expect(verifiedDiscordBody(signedRequest('not json'))).rejects.toMatchObject({
status: 400
});
await expect(verifiedDiscordBody(signedRequest([]))).rejects.toMatchObject({ status: 400 });
});
it('requires the configured application and a real guild member identity', () => {
expect(validInteractionEnvelope(interaction())).toBe(true);
expect(validInteractionEnvelope({ ...interaction(), application_id: guildId })).toBe(false);
expect(validInteractionEnvelope({ ...interaction(), guild_id: undefined })).toBe(false);
expect(
validInteractionEnvelope({ ...interaction(), member: {}, user: { id: userId } })
).toBe(false);
});
it('does not dispatch disabled or foreign-guild interactions', async () => {
const { POST } = await import('../../src/routes/api/discord/interactions/+server.js');
const foreign = await POST({
request: signedRequest({ ...interaction(), guild_id: userId })
});
expect((await foreign.json()).data.flags).toBe(64);
settings.enabled = false;
expect(
(await (await POST({ request: signedRequest(interaction()) })).json()).data.flags
).toBe(64);
expect(handle).not.toHaveBeenCalled();
});
it('defers privately, dispatches once per interaction and delivers without bot credentials', async () => {
const fetcher = vi.fn().mockResolvedValue(new Response('{}'));
vi.stubGlobal('fetch', fetcher);
const { POST } = await import('../../src/routes/api/discord/interactions/+server.js');
const pending = { type: 5, data: { flags: 64 } };
expect(await (await POST({ request: signedRequest(interaction()) })).json()).toEqual(
pending
);
expect(await (await POST({ request: signedRequest(interaction()) })).json()).toEqual(
pending
);
expect(handle).not.toHaveBeenCalled();
await new Promise((resolve) => setImmediate(resolve));
expect(handle).toHaveBeenCalledTimes(1);
expect(fetcher).toHaveBeenCalledOnce();
const [url, init] = fetcher.mock.calls[0];
expect(url).toContain(
`/webhooks/${applicationId}/private-interaction-token/messages/@original`
);
expect(init.headers.Authorization).toBeUndefined();
expect(JSON.parse(init.body).allowed_mentions).toEqual({ parse: [] });
});
it('prevents recipient mentions and reports sanitized delivery errors', async () => {
const fetcher = vi
.fn()
.mockResolvedValue(new Response('secret error body', { status: 403 }));
await expect(
completeDiscordInteraction(
applicationId,
'secret/token',
{ content: '@everyone', allowed_mentions: { parse: ['everyone'] } },
fetcher
)
).rejects.toThrow('Discord interaction delivery failed (403).');
const [url, init] = fetcher.mock.calls[0];
expect(url).toContain('secret%2Ftoken');
expect(JSON.parse(init.body).allowed_mentions.parse).toEqual([]);
});
it('honors short rate limits and rechecks publication before retrying', async () => {
vi.useFakeTimers();
let published = true;
const fetcher = vi
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ retry_after: 2 }), { status: 429 })
)
.mockResolvedValueOnce(new Response('{}'));
const delivery = completeDiscordInteraction(
applicationId,
'token',
{
content: 'Private player names',
allowed_mentions: { parse: [] }
},
fetcher,
() => published
);
await vi.advanceTimersByTimeAsync(1999);
expect(fetcher).toHaveBeenCalledTimes(1);
published = false;
await vi.advanceTimersByTimeAsync(1);
await delivery;
expect(fetcher).toHaveBeenCalledTimes(2);
const payload = fetcher.mock.calls[1][1].body;
expect(payload).not.toContain('Private player names');
expect(payload).toContain('Publication settings changed');
});
});
describe('Discord configuration and access', () => {
it('defaults to no publication and validates IDs and an explicit bounded allowlist', () => {
expect(validateDiscordSettings(defaultDiscordSettings)).toEqual(defaultDiscordSettings);
expect(defaultDiscordSettings.enabled).toBe(false);
expect(defaultDiscordSettings.showPlayers).toBe(false);
expect(defaultDiscordSettings.showAddresses).toBe(false);
expect(defaultDiscordSettings.leaderboardEnabled).toBe(false);
expect(() =>
validateDiscordSettings({ ...defaultDiscordSettings, enabled: true })
).toThrow();
expect(() =>
validateDiscordSettings({ ...defaultDiscordSettings, guildId: 'bad' })
).toThrow();
expect(() =>
validateDiscordSettings({
...defaultDiscordSettings,
publishedServers: ['../../secrets']
})
).toThrow();
expect(() =>
validateDiscordSettings({
...defaultDiscordSettings,
publishedServers: Array(101).fill('t5_map_1')
})
).toThrow();
expect(
validateDiscordSettings({
...defaultDiscordSettings,
publishedServers: ['t5_map_2', 't5_map_1', 't5_map_1']
}).publishedServers
).toEqual(['t5_map_1', 't5_map_2']);
});
it('limits browser configuration writes to administrators and exempts only exact signed POSTs', () => {
for (const route of [
'/api/integrations/discord',
'/api/integrations/discord/register',
'/settings/integrations/discord'
]) {
expect(routePermission(route, 'GET')).toBe('admin-read');
expect(roleAllows('super_viewer', routePermission(route, 'PUT'))).toBe(false);
expect(roleAllows('operator', routePermission(route, 'GET'))).toBe(false);
expect(roleAllows('admin', routePermission(route, 'PUT'))).toBe(true);
}
expect(routePermission('/api/discord/interactions', 'POST')).toBe('machine');
expect(routePermission('/api/discord/interactions', 'GET')).not.toBe('machine');
expect(routePermission('/api/discord/interactions/other', 'POST')).not.toBe('machine');
expect(routePermission('/api/discord/interactions-extra', 'POST')).not.toBe('machine');
});
});
+17 -2
View File
@@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
vi.mock('../../src/lib/server/db/migrate.js', () => ({ migrate: vi.fn() }));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ startWorker: vi.fn() }));
vi.mock('../../src/lib/server/discord/worker.js', () => ({ startDiscordWorker: vi.fn() }));
vi.mock('../../src/lib/server/provisioning/history.js', () => ({ interruptDeployments: vi.fn() }));
vi.mock('../../src/lib/server/auth/guard.js', () => ({
authenticationHandle: vi.fn(async () => new Response('OK'))
@@ -24,7 +25,13 @@ afterEach(() => {
});
describe('controller error handling', () => {
it.each(['migration', 'deployment recovery', 'worker startup', 'performance monitor'] as const)(
it.each([
'migration',
'deployment recovery',
'worker startup',
'Discord startup',
'performance monitor'
] as const)(
'retries failed %s before serving requests and boots only once after recovery',
async (phase) => {
const { handle } = await import('../../src/hooks.server.js');
@@ -32,13 +39,21 @@ describe('controller error handling', () => {
const { interruptDeployments } =
await import('../../src/lib/server/provisioning/history.js');
const { startWorker } = await import('../../src/lib/server/pipeline/worker.js');
const { startDiscordWorker } = await import('../../src/lib/server/discord/worker.js');
const { startPerformanceMonitor } = await import('../../src/lib/server/performance.js');
const { authenticationHandle } = await import('../../src/lib/server/auth/guard.js');
const steps = [migrate, interruptDeployments, startWorker, startPerformanceMonitor];
const steps = [
migrate,
interruptDeployments,
startWorker,
startDiscordWorker,
startPerformanceMonitor
];
const failedStep = [
'migration',
'deployment recovery',
'worker startup',
'Discord startup',
'performance monitor'
].indexOf(phase);
const failure = new Error('database or disk is full');