diff --git a/README.md b/README.md index 6d2dfb0..9593b5e 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,40 @@ server actions or send browser push notifications after the app closes. Plugins support named release versions, GitHub DLL imports and update discovery. See [Plugin releases](docs/plugin-releases.md) for the import and beta workflow. +## Plutonium updates and scheduled restarts + +Open **Settings → Plutonium updates & restarts** to check the latest published +revision, inspect each server's running revision, and queue a server or fleet +restart. Release and running-version checks also run hourly. The published +revision comes from the [official changelog](https://plutonium.pw/docs/changelog/); +running revisions come from RCON `version`. Unsupported replies and failed +queries appear as unknown rather than being treated as up to date. A failed +changelog request retains the last release with a visible error. + +Daily fleet restarts are optional and disabled by default. Choose a **UTC** queue +time and save the schedule; the next run is also shown in your display timezone. +Every due run queues the current fleet, regardless of whether an update exists. +The schedule and jobs survive app restarts. Missed days become one run, and a +server can have only one pending maintenance restart. Disabling the schedule +does not cancel work already queued; use **Cancel** for those jobs. + +Maintenance waits for a fresh zero-player RCON response and for existing +deployment/beta work, then uses the shared rolling restart limit. Unreachable or +occupied servers remain queued. After container and RCON recovery, Electric +Cherry queries the running revision again. If a fresh published revision was +known when queued, recovery on an older or unknown revision is reported as a +failed verification, without an automatic restart loop. Ambiguous restart +failures likewise require inspection before a new request. Activity is retained +for 30 days, with pending jobs and the latest 100 records shown in Settings. + +Updates rely on Plutainer's startup updater (`PLUTAINER_AUTO_UPDATE` must be +enabled); Electric Cherry does not download binaries or pull a container image. +See [Plutainer's startup flow](https://github.com/Ayymoss/Plutainer/blob/main/CLAUDE.md) +and [environment migration reference](https://github.com/Ayymoss/Plutainer/blob/main/MIGRATION.md). +The worker starts on the first application request, so keep the app running and +reachable for scheduled maintenance. Validate RCON revision replies and updater +behavior on your live T5/T6 servers before relying on unattended maintenance. + ## How restarts stay safe Betas support configurable gameplay, participation, stability and manual-review diff --git a/src/lib/plutonium.ts b/src/lib/plutonium.ts new file mode 100644 index 0000000..f90efde --- /dev/null +++ b/src/lib/plutonium.ts @@ -0,0 +1,55 @@ +export interface MaintenanceSchedule { + enabled: boolean; + time: string; + nextAt: string | null; +} +export interface PlutoniumRelease { + revision: number | null; + checkedAt: string | null; + error: string | null; +} +export interface PlutoniumVersion { + server: string; + revision: number | null; + checked_at: string | null; + error: string | null; +} +export interface MaintenanceJob { + id: number; + server: string; + state: 'queued' | 'restarting' | 'done' | 'failed' | 'cancelled'; + target_revision: number | null; + requested_at: string; + restarted_at: string | null; + finished_at: string | null; + detail: string | null; +} +export interface PlutoniumStatus { + release: PlutoniumRelease; + schedule: MaintenanceSchedule; + servers: PlutoniumVersion[]; + jobs: MaintenanceJob[]; +} + +/** Accept explicit revision tokens only, never engine versions or build dates. */ +export function parsePlutoniumRevision(text: string): number | null { + const match = text.replace(/\^[0-9]/g, '').match(/\br([1-9]\d{2,8})\b/i); + return match ? Number(match[1]) : null; +} + +export function parseChangelogRevision(html: string): number { + const headings = [...html.matchAll(/]*>([\s\S]*?)<\/h2>/gi)]; + const revisions = headings.map(h => parsePlutoniumRevision(h[1].replace(/<[^>]*>/g, ' '))) + .filter((r): r is number => r !== null); + if (!revisions.length) throw new Error('The Plutonium changelog did not contain a release heading.'); + return Math.max(...revisions); +} + +export function nextMaintenanceAt(time: string, now = new Date()): string { + if (!/^([01]\d|2[0-3]):[0-5]\d$/.test(time)) throw new Error('Choose a valid UTC time (HH:mm).'); + const next = new Date(now); + const [hour, minute] = time.split(':').map(Number); + next.setUTCHours(hour, minute, 0, 0); + if (next <= now) next.setUTCDate(next.getUTCDate() + 1); + return next.toISOString(); +} diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 81889dd..7743733 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -362,6 +362,24 @@ CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS plutonium_versions ( + server TEXT PRIMARY KEY REFERENCES servers(name) ON DELETE CASCADE, + revision INTEGER, + checked_at TEXT NOT NULL, + error TEXT +); +CREATE TABLE IF NOT EXISTS maintenance_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + server TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'queued', + target_revision INTEGER, + requested_at TEXT NOT NULL, + restarted_at TEXT, + finished_at TEXT, + detail TEXT +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_maintenance_pending ON maintenance_jobs(server) + WHERE state IN ('queued', 'restarting'); CREATE TABLE IF NOT EXISTS server_telemetry ( server TEXT PRIMARY KEY REFERENCES servers(name) ON DELETE CASCADE, round INTEGER NOT NULL, diff --git a/src/lib/server/fleet/plutonium.ts b/src/lib/server/fleet/plutonium.ts new file mode 100644 index 0000000..9822e48 --- /dev/null +++ b/src/lib/server/fleet/plutonium.ts @@ -0,0 +1,173 @@ +import { db } from '../db/index.js'; +import { config, komodoConfigured, rconConfigured } from '../config.js'; +import { getServer, listServers } from './servers.js'; +import { rconGetPlayerCount, rconPlutoniumVersion } from '../integrations/rcon.js'; +import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from '../pipeline/restart-rollout.js'; +import { nextMaintenanceAt, parseChangelogRevision } from '../../plutonium.js'; +import type { MaintenanceJob, MaintenanceSchedule, PlutoniumRelease, PlutoniumStatus, PlutoniumVersion } from '../../plutonium.js'; + +const CHANGELOG = 'https://plutonium.pw/docs/changelog/'; +const HOUR = 60 * 60 * 1000; +const nowIso = () => new Date().toISOString(); +function readSetting(key: string, fallback: T): T { + const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined; + return row ? JSON.parse(row.value) : fallback; +} +function writeSetting(key: string, value: unknown): void { + db().prepare('INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value') + .run(key, JSON.stringify(value)); +} +function release(): PlutoniumRelease { + return readSetting('plutonium_release', { revision: null, checkedAt: null, error: null }); +} +function schedule(): MaintenanceSchedule { + return readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null }); +} +export function saveMaintenanceSchedule(enabled: boolean, time: string): void { + const nextAt = nextMaintenanceAt(time); + writeSetting('plutonium_schedule', { enabled, time, nextAt: enabled ? nextAt : null }); +} +export function plutoniumStatus(): PlutoniumStatus { + const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[]; + return { + release: release(), schedule: schedule(), + servers: listServers().map(s => versions.find(v => v.server === s.name) + ?? { server: s.name, revision: null, checked_at: null, error: null }), + jobs: db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') + OR id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) ORDER BY id DESC`).all() as MaintenanceJob[] + }; +} + +async function probeVersion(server: string): Promise { + const recovering = db().prepare('SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(server); + const result = recovering ? { revision: null, error: 'Waiting for restart recovery.' } + : await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' })); + // Clear the old revision on failure: it is no longer evidence of what is running. + db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?) + ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`) + .run(server, result.revision, nowIso(), result.error); + return result.revision; +} + +let checking: Promise | null = null; +export function checkPlutonium(): Promise { + if (checking) return checking; + checking = check().finally(() => { checking = null; }); + return checking; +} +async function check(): Promise { + const previous = release(); + try { + const response = await fetch(CHANGELOG, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' }); + if (!response.ok) throw new Error(`Changelog request failed (HTTP ${response.status}).`); + const html = await response.text(); + if (html.length > 2_000_000) throw new Error('Changelog response was too large.'); + writeSetting('plutonium_release', { revision: parseChangelogRevision(html), checkedAt: nowIso(), error: null }); + } catch (err) { + writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message }); + } + const servers = listServers(); + for (let i = 0; i < servers.length; i += 4) { + await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name))); + } +} + +export function queueMaintenance(servers: string[]): number { + const names = new Set(listServers().map(s => s.name)); + if (!servers.length || servers.some(s => !names.has(s))) throw new Error('Choose existing fleet servers.'); + return db().transaction(() => { + const latest = release(); + // An old cached release is still displayed, but never used as a fresh target. + const target = !latest.error && latest.checkedAt && Date.now() - Date.parse(latest.checkedAt) < HOUR + ? latest.revision : null; + let added = 0; + for (const server of new Set(servers)) { + added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at) + VALUES (?, ?, ?)`).run(server, target, nowIso()).changes; + } + return added; + })(); +} +export function cancelMaintenance(id: number): boolean { + return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator' + WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0; +} + +function conflictingWork(server: string): boolean { + return !!db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND state IN ('queued','running','awaiting-health') + UNION ALL SELECT 1 FROM deployment_servers WHERE server = ? AND restart_state IN ('pending','failed') + UNION ALL SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id + WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server, server, server); +} +function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) { + db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?') + .run(state, detail, nowIso(), job.id); +} +function hold(job: MaintenanceJob, detail: string) { + db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ?').run(detail, job.id); +} + +let ticking: Promise | null = null; +export function maintenanceTick(): Promise { + if (ticking) return ticking; + ticking = runTick().finally(() => { ticking = null; }); + return ticking; +} +async function runTick(): Promise { + const latest = release(); + if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR) await checkPlutonium(); + db().transaction(() => { + const plan = schedule(); + if (plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) { + const names = listServers().map(s => s.name); + if (names.length) queueMaintenance(names); + // Coalesce missed days after downtime into one run; no restart backlog. + writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time) }); + } + })(); + await refreshRestartRollout(); + const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[]; + for (const job of jobs) { + try { await processJob(job); } + catch { hold(job, 'Maintenance check failed; waiting for the next check.'); } + } + db().prepare(`DELETE FROM maintenance_jobs WHERE finished_at < ? AND state IN ('done','failed','cancelled')`) + .run(new Date(Date.now() - 30 * 24 * HOUR).toISOString()); +} +async function processJob(job: MaintenanceJob): Promise { + const server = getServer(job.server); + if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.'); + if (job.state === 'restarting') { + const slot = db().prepare('SELECT requested_at FROM restart_rollout WHERE server = ?').get(job.server) as { requested_at: string } | undefined; + if (!slot || !job.restarted_at || slot.requested_at < job.restarted_at) { + return finish(job, 'failed', 'Restart submission was interrupted. Inspect the server before queuing another restart.'); + } + const health = rolloutHealth(job.server); + if (!health.ok) return hold(job, health.reason); + const revision = await probeVersion(job.server); + if (job.target_revision && (revision === null || revision < job.target_revision)) { + return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`); + } + return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`); + } + if (!komodoConfigured() || !rconConfigured()) return hold(job, 'Configure Komodo and RCON before restarting.'); + if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.'); + const reason = rolloutWaitReason(job.server); + if (reason) return hold(job, reason); + const players = await rconGetPlayerCount(job.server); + if (players === null || players > 0) return hold(job, players === null ? 'Waiting for a fresh RCON player count.' : `Waiting for ${players} player(s) to leave.`); + // Recheck after async I/O; cancellation and lifecycle operations may have arrived. + const current = getServer(job.server); + if (!current || current.destroyed_at) return finish(job, 'cancelled', 'Server removed.'); + if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.'); + const claimed = db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ?, detail = 'Restart requested' + WHERE id = ? AND state = 'queued'`).run(nowIso(), job.id); + if (!claimed.changes) return; + db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?') + .run('Waiting for restart recovery.', job.server); + // Persist intent before remote I/O. An ambiguous timeout must never cause an automatic second restart. + const result = await restartInRollout(job.server); + if (result.kind === 'queued') { + db().prepare(`UPDATE maintenance_jobs SET state = 'queued', restarted_at = NULL, detail = ? WHERE id = ?`).run(result.message, job.id); + } else if (result.kind === 'failed') finish(job, 'failed', result.message); +} diff --git a/src/lib/server/integrations/rcon.ts b/src/lib/server/integrations/rcon.ts index 947f0d0..83a2887 100644 --- a/src/lib/server/integrations/rcon.ts +++ b/src/lib/server/integrations/rcon.ts @@ -3,6 +3,7 @@ import type { Socket } from 'node:dgram'; import { config, komodoConfigured, rconConfigured } from '../config.js'; import { getServer, setServerField } from '../fleet/servers.js'; import { komodoResolveGamePort } from './komodo.js'; +import { parsePlutoniumRevision } from '../../plutonium.js'; // Plutonium/CoD RCON is UDP with a 4-byte 0xff prefix. Requests look like // \xff\xff\xff\xffrcon @@ -451,3 +452,14 @@ function positiveCapacity(value: string | undefined): number | null { const count = Number(value); return Number.isInteger(count) && count > 0 ? count : null; } + +/** Query the running process, not the updater's on-disk files. */ +export async function rconPlutoniumVersion(server: string): Promise<{ revision: number | null; error: string | null }> { + const { endpoint } = await endpointFor(server); + if (!endpoint) return { revision: null, error: 'No RCON endpoint available.' }; + const response = await sendRcon(endpoint.host, endpoint.port, 'version'); + if (!response.ok) return { revision: null, error: response.detail }; + const body = stripPrefix(response.buf); + const revision = body?.trimStart().startsWith('print') ? parsePlutoniumRevision(body) : null; + return { revision, error: revision === null ? 'RCON version reply did not identify a Plutonium revision.' : null }; +} diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index f9ece10..99b6789 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -13,6 +13,7 @@ import { isProvisioning } from '../fleet/provisioning.js'; import { STATUS_REFRESH_MS } from '../../server-freshness.js'; import { recordMetrics } from '../fleet/metrics.js'; import { selectedZombieMap } from '../../map-aliases.js'; +import { maintenanceTick } from '../fleet/plutonium.js'; const TICK_MS = 15_000; // Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the @@ -46,6 +47,8 @@ async function runTick(): Promise { } catch (err) { console.error('[worker] beta-runner error:', err); } + try { await maintenanceTick(); } + catch (err) { console.error('[worker] maintenance error:', err); } } /** diff --git a/src/lib/ui/components/PlutoniumMaintenance.svelte b/src/lib/ui/components/PlutoniumMaintenance.svelte new file mode 100644 index 0000000..b54e8cc --- /dev/null +++ b/src/lib/ui/components/PlutoniumMaintenance.svelte @@ -0,0 +1,108 @@ + + +
+

Plutonium updates & restarts

+ +
+

Plutainer applies updates on startup when auto-update is enabled. Restarts wait for a fresh player count of zero and share the fleet’s rolling restart limit.

+ {#if error}{/if} + {#if notice}

{notice}

{/if} + {#if data} +

Latest published revision: {data.release.revision ? `r${data.release.revision}` : 'Unknown'} + · Changelog + {#if data.release.checkedAt} · Checked {$timeFormat.dateTime(data.release.checkedAt)}{/if} +

+ {#if data.release.error}

Update check failed: {data.release.error} Any displayed release is cached.

{/if} +
{ event.preventDefault(); void refresh({ action: 'schedule', enabled, time }); }}> + + + +
+

This restarts all fleet servers daily, even without a new release. Busy servers may restart later. Disabling the schedule stops future runs; cancel existing queued restarts below.

+ {#if data.schedule.nextAt}

Next run: {$timeFormat.dateTime(data.schedule.nextAt)}

{/if} +
+ + {#each data.servers as server (server.server)} + + + + + {:else}{/each} +
ServerRunning revisionUpdate statusAction
{server.server}{server.revision ? `r${server.revision}` : 'Unknown'} + {#if server.checked_at}Checked {$timeFormat.dateTime(server.checked_at)}{/if} + {#if server.error}{server.error}{/if} + {server.revision === null || data.release.revision === null || data.release.error ? 'Unverified' + : server.revision < data.release.revision ? 'Update available' : server.revision > data.release.revision ? 'Ahead of changelog' : 'Up to date'}
No fleet servers yet.
+ {#if data.servers.length}{/if} + {#if data.jobs.length} +

Restart activity

+
    {#each data.jobs as job (job.id)}
  • + {job.server} · {job.state} · {$timeFormat.dateTime(job.requested_at)} + {#if job.target_revision} · Target r{job.target_revision}{/if} + {#if job.detail}{job.detail}{/if} + {#if job.state === 'queued'}{/if} +
    • {/each}
+ {/if} + {:else}

{busy ? 'Loading maintenance status…' : 'Maintenance status unavailable.'}

{/if} +
+ + diff --git a/src/routes/api/plutonium/+server.ts b/src/routes/api/plutonium/+server.ts new file mode 100644 index 0000000..296a728 --- /dev/null +++ b/src/routes/api/plutonium/+server.ts @@ -0,0 +1,27 @@ +import { json } from '@sveltejs/kit'; +import { cancelMaintenance, checkPlutonium, plutoniumStatus, queueMaintenance, saveMaintenanceSchedule } from '$lib/server/fleet/plutonium.js'; + +export function GET() { return json(plutoniumStatus()); } + +export async function POST({ request }: { request: Request }) { + let body; + try { body = await request.json(); } catch { return json({ error: 'Invalid JSON.' }, { status: 400 }); } + if (!body || typeof body !== 'object') return json({ error: 'Invalid request.' }, { status: 400 }); + if (body.action === 'check') { + await checkPlutonium(); + } else if (body.action === 'schedule') { + if (typeof body.enabled !== 'boolean' || typeof body.time !== 'string' || !/^([01]\d|2[0-3]):[0-5]\d$/.test(body.time)) { + return json({ error: 'Provide enabled and a valid UTC time (HH:mm).' }, { status: 400 }); + } + saveMaintenanceSchedule(body.enabled, body.time); + } else if (body.action === 'restart') { + if (!Array.isArray(body.servers) || !body.servers.length || body.servers.some((s: unknown) => typeof s !== 'string')) { + return json({ error: 'Choose servers to restart.' }, { status: 400 }); + } + try { queueMaintenance(body.servers); } + catch (err) { return json({ error: (err as Error).message }, { status: 400 }); } + } else if (body.action === 'cancel') { + if (!Number.isSafeInteger(body.id) || !cancelMaintenance(body.id)) return json({ error: 'Only queued restarts can be cancelled.' }, { status: 409 }); + } else return json({ error: 'Unknown action.' }, { status: 400 }); + return json(plutoniumStatus()); +} diff --git a/src/routes/settings/+page.svelte b/src/routes/settings/+page.svelte index 35077c5..f1e4f32 100644 --- a/src/routes/settings/+page.svelte +++ b/src/routes/settings/+page.svelte @@ -6,6 +6,7 @@ import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Button from '$ui/components/primitives/Button.svelte'; import Badge from '$ui/components/primitives/Badge.svelte'; + import PlutoniumMaintenance from '$ui/components/PlutoniumMaintenance.svelte'; let data = $state(null); let loading = $state(false); let error = $state(''); @@ -38,6 +39,7 @@ {#if data}

Example: {$timeFormat.dateTime(data.checkedAt)}

{/if} {#if $timeZonePersistenceError}

Browser storage is unavailable. Your timezone applies for this visit but cannot be saved.

{/if} +

System status

Check your connections and the prerequisites for a playable server.

diff --git a/tests/unit/plutonium.test.ts b/tests/unit/plutonium.test.ts new file mode 100644 index 0000000..3af8c73 --- /dev/null +++ b/tests/unit/plutonium.test.ts @@ -0,0 +1,151 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { nextMaintenanceAt, parseChangelogRevision, parsePlutoniumRevision } from '../../src/lib/plutonium.js'; + +vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() })); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() })); + +let fixture: Fixture; +const servers = ['t5_zombie_theater_1', 't6_zombie_town_1']; +beforeEach(async () => { + vi.resetModules(); vi.resetAllMocks(); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-09-21T05:59:00Z')); + fixture = await newFixture(servers); + vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); + vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test'); + vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); + vi.stubGlobal('fetch', vi.fn().mockImplementation(async () => new Response('

r5354

r5300

'))); + const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); + await seedServerRegistry(); + const k = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(k.komodoRestart).mockResolvedValue({ ok: true }); + vi.mocked(k.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + const r = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0); + vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null }); + vi.mocked(r.rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '', status: { players: 0, map: null, gametype: null, roster: [] } }); +}); +afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.useRealTimers(); }); +const api = () => import('../../src/lib/server/fleet/plutonium.js'); + +describe('Plutonium versions', () => { + it('parses explicit revisions and release headings without interpreting engine numbers or links as releases', () => { + expect(parsePlutoniumRevision('print\nPlutonium T6 r5354 win-x86')).toBe(5354); + expect(parsePlutoniumRevision('version 1.0 build 20260921')).toBeNull(); + expect(parseChangelogRevision('r9999

r5354

')).toBe(5354); + expect(() => parseChangelogRevision('

Service unavailable

')).toThrow(); + }); + it('preserves the cached release with a visible error and clears failed running-version observations', async () => { + const m = await api(); await m.checkPlutonium(); + vi.mocked(fetch).mockRejectedValue(new Error('Offline')); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Timeout' }); + await m.checkPlutonium(); + expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: 'Offline' }); + expect(m.plutoniumStatus().servers.every(s => s.revision === null && s.error === 'Timeout')).toBe(true); + }); +}); + +describe('persistent maintenance', () => { + it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => { + const m = await api(); await m.checkPlutonium(); + expect(m.queueMaintenance([servers[0]])).toBe(1); + expect(m.queueMaintenance([servers[0]])).toBe(0); + const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null); + await m.maintenanceTick(); await m.maintenanceTick(); + expect(komodoRestart).not.toHaveBeenCalled(); + await Promise.all([m.maintenanceTick(), m.maintenanceTick()]); + expect(komodoRestart).toHaveBeenCalledTimes(1); + expect(m.plutoniumStatus().jobs[0].state).toBe('restarting'); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0].state).toBe('restarting'); + vi.setSystemTime(Date.now() + 15_000); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' }); + }); + it('shares rolling capacity across servers and survives closing the database', async () => { + const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers); + await m.maintenanceTick(); + const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); + expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']); + vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']); + }); + it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => { + const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + await m.maintenanceTick(); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null }); + vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick(); await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 }); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + expect(komodoRestart).toHaveBeenCalledTimes(1); + }); + it('does not retry an ambiguous restart timeout', async () => { + const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' }); + await m.maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0].state).toBe('failed'); + expect(komodoRestart).toHaveBeenCalledTimes(1); + }); + it('honors cancellation arriving during the player check', async () => { + const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { + m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0; + }); + await m.maintenanceTick(); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + expect(komodoRestart).not.toHaveBeenCalled(); + expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled'); + }); + it('holds maintenance while lifecycle work is queued', async () => { + const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString()); + await m.maintenanceTick(); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + expect(komodoRestart).not.toHaveBeenCalled(); + expect(m.plutoniumStatus().jobs[0].detail).toContain('deployment or beta'); + }); + it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => { + const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run(); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs[0].state).toBe('failed'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + expect(komodoRestart).not.toHaveBeenCalled(); + }); + it('validates API requests without changing the schedule or queue', async () => { + const { POST } = await import('../../src/routes/api/plutonium/+server.js'); + const post = (body: unknown) => POST({ request: new Request('http://localhost/api/plutonium', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) + }) }); + expect((await post({ action: 'schedule', enabled: true, time: '26:99' })).status).toBe(400); + expect((await post({ action: 'restart', servers: ['missing'] })).status).toBe(400); + expect((await post({ action: 'cancel', id: 100 })).status).toBe(409); + expect((await post({ action: 'schedule', enabled: true, time: '06:00' })).status).toBe(200); + const m = await api(); + expect(m.plutoniumStatus().jobs).toEqual([]); + expect(m.plutoniumStatus().schedule.enabled).toBe(true); + }); + it('runs a daily UTC schedule once and coalesces missed days', async () => { + expect(nextMaintenanceAt('06:00', new Date('2026-09-21T06:00:00Z'))).toBe('2026-09-22T06:00:00.000Z'); + expect(() => nextMaintenanceAt('24:00')).toThrow(); + const m = await api(); await m.checkPlutonium(); + expect(m.plutoniumStatus().schedule.enabled).toBe(false); + m.saveMaintenanceSchedule(true, '06:00'); + vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); + await m.maintenanceTick(); await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toHaveLength(2); + expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z'); + m.saveMaintenanceSchedule(false, '06:00'); + expect(m.plutoniumStatus().schedule.nextAt).toBeNull(); + }); +});