Roll deployment restarts through a shared fleet-wide recovery limit

.env.example

@@ -10,6 +10,10 @@ DB_PATH=/data/cherry.db
 # Retention (days) for deployment rows and blob diffs
 DEPLOY_KEEP_DAYS=7
 
+# Fleet-wide rolling restart limit, shared by deployments and betas.
+# A slot stays occupied until Komodo and RCON confirm recovery.
+DEPLOYMENT_RESTART_CONCURRENCY=1
+
 # Komodo
 KOMODO_ADDRESS=https://komodo.example.internal
 KOMODO_API_KEY=

README.md

@@ -31,13 +31,26 @@ audit diffs.
 
 ## How restarts stay safe
 
-Every push (or beta open) creates a `deployment` row and marks each affected
+File changes are staged sequentially. Their activation uses rolling restarts:
-server `pending`. A background sweep runs every 15s and:
+by default only **one server across the fleet** can restart or recover at a
+time. Promotions, rollbacks, drift reconciliation, and beta restarts share this
+limit. Set `DEPLOYMENT_RESTART_CONCURRENCY` to a positive integer to raise it.
 
-1. Asks each server over RCON: how many players are on right now?
+A background sweep runs every 15s. It checks recovering servers first, then
-2. If 0 → tell Komodo to restart the server → row goes `done`.
+admits queued deployments in oldest-first order when there is capacity. Busy
-3. If >0 → hold the row `pending` with the player count; try again next tick.
+or unreachable servers stay queued without taking a slot, so other empty
-4. If RCON unreachable → hold `pending` with `error='rcon unreachable'`.
+servers can proceed. A successful Komodo restart request retains its slot
+until the container is healthy and RCON answers. Health checks start no sooner
+than 15 seconds after the request returns, avoiding an immediate check of the
+old process. An unhealthy server keeps its slot rather than letting failures
+spread across the fleet. A timed-out restart also keeps its slot because Komodo
+may have accepted the request.
+
+Slots persist in SQLite across app restarts, and overlapping timer/API triggers
+share the active run. Beta options to skip health or match gates still work;
+skipping a beta gate does not release the fleet's recovery slot early. New-server
+creation is separate from deployment restarts and still starts its container
+directly.
 
 No token, no session, no auth freeze. If Komodo or RCON aren't configured,
 the sweep marks pending rows `manual` so nothing hangs forever.

docker-compose.yml

@@ -31,6 +31,7 @@ services:
       BETA_ARCHIVE_DIR: /root/zombies/beta-archive
       DB_PATH: /data/cherry.db
       DEPLOY_KEEP_DAYS: "7"
+      DEPLOYMENT_RESTART_CONCURRENCY: ${DEPLOYMENT_RESTART_CONCURRENCY:-1}
       KOMODO_ADDRESS: ${KOMODO_ADDRESS}
       KOMODO_API_KEY: ${KOMODO_API_KEY}
       KOMODO_API_SECRET: ${KOMODO_API_SECRET}

komodo/stack.toml

@@ -26,6 +26,7 @@ REL_DEST         = "runtime/plutonium/storage/t5/raw/scripts/sp"
 BETA_ARCHIVE_DIR = "/root/zombies/beta-archive"
 DB_PATH          = "/data/cherry.db"
 DEPLOY_KEEP_DAYS = "7"
+DEPLOYMENT_RESTART_CONCURRENCY = "1"
 
 # secrets: bind via Komodo's env-file or secret store
 KOMODO_ADDRESS   = "https://komodo.example.internal"

src/lib/server/config.ts

@@ -14,6 +14,12 @@ function num(name: string, fallback: number): number {
     return n;
 }
 
+function positiveInt(name: string, fallback: number): number {
+    const value = num(name, fallback);
+    if (!Number.isSafeInteger(value) || value < 1) throw new Error(`env ${name} must be a positive integer`);
+    return value;
+}
+
 export const config = {
     serversRoot: req('SERVERS_ROOT', '/root/zombies/t5'),
     relDest: req('REL_DEST', 'runtime/plutonium/storage/t5/raw/scripts/sp'),
@@ -29,6 +35,7 @@ export const config = {
     betaArchiveDir: req('BETA_ARCHIVE_DIR', '/root/zombies/beta-archive'),
     dbPath: req('DB_PATH', './data/cherry.db'),
     deployKeepDays: num('DEPLOY_KEEP_DAYS', 7),
+    restartConcurrency: positiveInt('DEPLOYMENT_RESTART_CONCURRENCY', 1),
     httpTimeoutMs: num('HTTP_TIMEOUT_MS', 10_000),
     komodo: {
         address: env.KOMODO_ADDRESS ?? '',

src/lib/server/db/schema.sql

@@ -199,6 +199,18 @@ CREATE TABLE IF NOT EXISTS deployment_servers (
 
 CREATE INDEX IF NOT EXISTS idx_deployment_servers_state ON deployment_servers(restart_state);
 
+-- Fleet-wide restart slots shared by deployments and betas. A successful API
+-- response does not release a slot: the server must recover first. Persisting
+-- these rows also protects the fleet across Electric Cherry process restarts.
+CREATE TABLE IF NOT EXISTS restart_rollout (
+    server       TEXT PRIMARY KEY REFERENCES servers(name) ON DELETE CASCADE,
+    requested_at TEXT NOT NULL,
+    checked_at   TEXT,
+    healthy_at   TEXT,
+    players      INTEGER,
+    error        TEXT
+);
+
 CREATE TABLE IF NOT EXISTS deployment_files (
     id             INTEGER PRIMARY KEY AUTOINCREMENT,
     deployment_id  TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE,

src/lib/server/pipeline/beta-runner.ts

@@ -2,9 +2,9 @@ import { db } from '../db/index.js';
 import { emit } from './bus.js';
 import type { BetaStage } from './bus.js';
 import { komodoConfigured, rconConfigured } from '../config.js';
-import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
+import { rconGetPlayerCount } from '../integrations/rcon.js';
-import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js';
 import { scanForMatchEnd } from '../fleet/match-log.js';
+import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
 
 interface BetaServerActiveRow {
     beta_id: number;
@@ -84,8 +84,17 @@ function reconcileBetaState(betaId: number): void {
     }
 }
 
-export async function betaTick(): Promise<{ processed: number }> {
+let betaInFlight: Promise<{ processed: number }> | null = null;
+
+export function betaTick(): Promise<{ processed: number }> {
+    if (betaInFlight) return betaInFlight;
+    betaInFlight = runBetaTick().finally(() => { betaInFlight = null; });
+    return betaInFlight;
+}
+
+async function runBetaTick(): Promise<{ processed: number }> {
     if (!integrationsOn()) return { processed: 0 };
+    await refreshRestartRollout();
 
     const rows = db()
         .prepare(
@@ -95,7 +104,8 @@ export async function betaTick(): Promise<{ processed: number }> {
              FROM beta_servers bs
              JOIN betas b ON b.id = bs.beta_id
              WHERE b.state = 'running'
-               AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')`
+               AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')
+             ORDER BY b.id, bs.server`
         )
         .all() as BetaServerActiveRow[];
 
@@ -117,19 +127,24 @@ export async function betaTick(): Promise<{ processed: number }> {
 
 async function advance(row: BetaServerActiveRow): Promise<void> {
     switch (row.stage) {
-        case 'awaiting-empty': {
+        case 'awaiting-empty':
+        case 'restarting': {
+            const waitReason = rolloutWaitReason(row.server);
+            if (waitReason) {
+                transition(row.beta_id, row.server, 'awaiting-empty', waitReason);
+                return;
+            }
             const players = await rconGetPlayerCount(row.server);
             if (players === null) return; // hold — RCON unreachable, try next tick
             if (players > 0) return;      // hold — still populated
             transition(row.beta_id, row.server, 'restarting');
-            // Fall through to restart on the same tick for lower latency.
+            const r = await restartInRollout(row.server);
-            await advance({ ...row, stage: 'restarting' });
+            if (r.kind === 'queued') {
-            return;
+                transition(row.beta_id, row.server, 'awaiting-empty', r.message);
-        }
+                return;
-        case 'restarting': {
+            }
-            const r = await komodoRestart(row.server);
+            if (r.kind === 'failed') {
-            if (!r.ok) {
+                transition(row.beta_id, row.server, 'failed', r.message);
-                transition(row.beta_id, row.server, 'failed', `komodo: ${r.message}`);
                 return;
             }
             const next = row.wait_for_health ? 'awaiting-health' : row.wait_for_match ? 'awaiting-match' : 'ready';
@@ -137,14 +152,13 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
             return;
         }
         case 'awaiting-health': {
-            const h = await komodoInspectHealthy(row.server);
+            const h = rolloutHealth(row.server);
             if (!h.ok) {
                 // Record the failure reason (including any snippet) so the
                 // beta detail UI can surface why we're stuck.
-                const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
                 setStage().run(
                     'awaiting-health',
-                    `unhealthy: ${h.reason}${snippet}`,
+                    h.reason,
                     nowIso(),
                     'awaiting-health',
                     'awaiting-health',
@@ -159,8 +173,6 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
                 );
                 return;
             }
-            const rcon = await rconStatusResult(row.server);
-            if (!rcon.ok) return; // container reports up but game not answering yet
             const next = row.wait_for_match ? 'awaiting-match' : 'ready';
             transition(row.beta_id, row.server, next);
             return;

src/lib/server/pipeline/restart-rollout.ts

@@ -0,0 +1,125 @@
+import { config } from '../config.js';
+import { db } from '../db/index.js';
+import { komodoInspectHealthy, komodoRestart } from '../integrations/komodo.js';
+import { rconStatusResult } from '../integrations/rcon.js';
+
+// Komodo queues restart work. Don't mistake the still-running old container
+// for a completed restart when an API kick immediately follows the request.
+const HEALTH_POLL_MS = 15_000;
+const requestsInFlight = new Set<string>();
+
+interface RestartSlot {
+    server: string;
+    requested_at: string;
+    checked_at: string | null;
+    healthy_at: string | null;
+    players: number | null;
+    error: string | null;
+}
+
+function slotFor(server: string): RestartSlot | undefined {
+    return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
+}
+
+export function rolloutWaitReason(server: string): string | null {
+    const slot = slotFor(server);
+    if (slot?.healthy_at === null) {
+        return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
+    }
+    const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
+    return n >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
+}
+
+export type RolloutRestartResult =
+    | { kind: 'started' }
+    | { kind: 'queued'; message: string }
+    | { kind: 'failed'; message: string };
+
+/** Reserve capacity before any async I/O; callers still check for players first. */
+export async function restartInRollout(server: string): Promise<RolloutRestartResult> {
+    const waitReason = db().transaction(() => {
+        const reason = rolloutWaitReason(server);
+        if (reason) return reason;
+        db().prepare(`INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)
+            ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
+                checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`)
+            .run(server, new Date().toISOString());
+        return null;
+    }).immediate();
+    if (waitReason) return { kind: 'queued', message: waitReason };
+
+    requestsInFlight.add(server);
+    try {
+        const result = await komodoRestart(server);
+        if (result.ok) return { kind: 'started' };
+        // These failures cannot have queued a remote restart. Other failures
+        // (especially a timeout) may have: retain capacity until recovery.
+        if (result.kind === 'no-name' || result.kind === 'unconfigured') {
+            db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server);
+        } else {
+            db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
+        }
+        return { kind: 'failed', message: `komodo: ${result.message}` };
+    } catch (err) {
+        const message = `komodo: ${(err as Error).message}`;
+        db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
+        return { kind: 'failed', message };
+    } finally {
+        // Start the grace period at completion, including slow/failed requests.
+        db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?')
+            .run(new Date().toISOString(), server);
+        requestsInFlight.delete(server);
+    }
+}
+
+let refreshInFlight: Promise<void> | null = null;
+
+/** Poll all occupied slots, even if their originating beta was already closed. */
+export function refreshRestartRollout(): Promise<void> {
+    if (refreshInFlight) return refreshInFlight;
+    refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
+    return refreshInFlight;
+}
+
+async function refresh(): Promise<void> {
+    // Adopt work started before this feature was installed. INSERT OR IGNORE
+    // preserves completed slots and never resets a newer restart's progress.
+    db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at)
+        SELECT server, MAX(restarted_at) FROM deployment_servers
+        WHERE restart_state IN ('pending','failed') AND restarted_at IS NOT NULL GROUP BY server`).run();
+    db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at)
+        SELECT bs.server, MAX(COALESCE(bs.restarted_at, bs.updated_at))
+        FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
+        WHERE b.state = 'running' AND bs.stage IN ('restarting','awaiting-health') GROUP BY bs.server`).run();
+
+    const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
+        .all() as RestartSlot[];
+    for (const slot of slots) {
+        if (requestsInFlight.has(slot.server)) continue;
+        if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
+        let error: string | null = null;
+        let players: number | null = null;
+        try {
+            const health = await komodoInspectHealthy(slot.server);
+            if (!health.ok) {
+                error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
+            } else {
+                const rcon = await rconStatusResult(slot.server);
+                if (!rcon.ok) error = `rcon not answering: ${rcon.reason}`;
+                else players = rcon.status.players;
+            }
+        } catch (err) {
+            error = `health check: ${(err as Error).message}`;
+        }
+        const now = new Date().toISOString();
+        db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
+            WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
+            .run(now, error ? null : now, players, error, slot.server, slot.requested_at);
+    }
+}
+
+export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
+    const slot = slotFor(server);
+    if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
+    return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' };
+}

src/lib/server/pipeline/sweep.ts

@@ -1,17 +1,21 @@
 import { config, komodoConfigured, rconConfigured } from '../config.js';
 import { db } from '../db/index.js';
 import { emit } from './bus.js';
-import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
+import { rconGetPlayerCount } from '../integrations/rcon.js';
-import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js';
 import { setServerField } from '../fleet/servers.js';
+import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
 
 interface PendingRow {
     deployment_id: string;
     server: string;
     attempts: number;
     restarted_at: string | null;
+    files_changed: number;
 }
 
+type SweepResult = { checked: number; restarted: number; healthy: number; held: number; failed: number };
+let sweepInFlight: Promise<SweepResult> | null = null;
+
 function nowIso(): string {
     return new Date().toISOString();
 }
@@ -35,7 +39,13 @@ function integrationsOn(): boolean {
  * Same-server dedup is per-tick per-phase so multiple concurrent deployments
  * targeting the same server only cost one RCON+Komodo round trip per tick.
  */
-export async function sweep(): Promise<{ checked: number; restarted: number; healthy: number; held: number; failed: number }> {
+export function sweep(): Promise<SweepResult> {
+    if (sweepInFlight) return sweepInFlight;
+    sweepInFlight = runSweep().finally(() => { sweepInFlight = null; });
+    return sweepInFlight;
+}
+
+async function runSweep(): Promise<SweepResult> {
     if (!integrationsOn()) {
         // Freeze pending rows as manual — they won't drain without integrations.
         db()
@@ -49,11 +59,15 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
         return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
     }
 
+    // Recovery frees capacity before admitting any more servers. This runs
+    // even without pending deployment rows (betas share the same slots).
+    await refreshRestartRollout();
     const pending = db()
         .prepare(
-            `SELECT deployment_id, server, attempts, restarted_at
+            `SELECT ds.deployment_id, ds.server, ds.attempts, ds.restarted_at, ds.files_changed
-             FROM deployment_servers
+             FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
-             WHERE restart_state IN ('pending','failed')`
+             WHERE ds.restart_state IN ('pending','failed')
+             ORDER BY d.started_at, d.id, ds.server`
         )
         .all() as PendingRow[];
     if (pending.length === 0) {
@@ -65,6 +79,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
     // Split dedup caches by phase — the two phases talk to different endpoints.
     const preRestart = new Map<string, Outcome>();
     const postRestart = new Map<string, Outcome>();
+    const recoveringServers = new Set(pending.filter((row) => row.restarted_at).map((row) => row.server));
 
     let checked = 0;
     let restarted = 0;
@@ -80,13 +95,18 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
     );
 
     for (const row of pending) {
+        // A failed file write isn't a restart request. Preserve the original
+        // failure until the operator repairs/retries that deployment.
+        if (row.files_changed === 0) continue;
         const cache = row.restarted_at ? postRestart : preRestart;
         let outcome = cache.get(row.server);
         if (!outcome) {
             checked++;
             outcome = row.restarted_at
-                ? await runHealthPhase(row.server)
+                ? runHealthPhase(row.server)
-                : await runRestartPhase(row.server);
+                : recoveringServers.has(row.server)
+                    ? { state: 'pending', players: null, error: 'Queued behind an earlier restart of this server', restartedNow: false }
+                    : await runRestartPhase(row.server);
             cache.set(row.server, outcome);
             if (outcome.state === 'done' && row.restarted_at) healthy++;
             if (outcome.restartedNow) restarted++;
@@ -152,6 +172,8 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
 
 // Phase 1: wait for the server to be empty, then Komodo-restart it.
 async function runRestartPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
+    const waitReason = rolloutWaitReason(server);
+    if (waitReason) return { state: 'pending', players: null, error: waitReason, restartedNow: false };
     const players = await rconGetPlayerCount(server);
     if (players === null) {
         return { state: 'pending', players: null, error: 'rcon unreachable', restartedNow: false };
@@ -159,32 +181,22 @@ async function runRestartPhase(server: string): Promise<{ state: 'done' | 'faile
     if (players > 0) {
         return { state: 'pending', players, error: null, restartedNow: false };
     }
-    const r = await komodoRestart(server);
+    const r = await restartInRollout(server);
-    if (r.ok) {
+    if (r.kind === 'started') {
         // Restart triggered — flip restarted_at so the next tick runs the
         // health phase. Still 'pending' overall until health confirms.
         return { state: 'pending', players: 0, error: null, restartedNow: true };
     }
-    if (r.kind === 'no-name') {
+    return { state: r.kind === 'queued' ? 'pending' : 'failed', players: 0, error: r.message, restartedNow: false };
-        return { state: 'failed', players: 0, error: r.message, restartedNow: false };
-    }
-    return { state: 'failed', players: 0, error: `komodo: ${r.message}`, restartedNow: false };
 }
 
 // Phase 2: after Komodo-restart, wait until the container reports running AND
 // RCON is answering. Only then does the deployment_server row transition to
 // 'done'. If either check fails we stay pending and try again next tick.
-async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
+function runHealthPhase(server: string): { state: 'done' | 'pending'; players: number | null; error: string | null; restartedNow: boolean } {
-    const h = await komodoInspectHealthy(server);
+    const h = rolloutHealth(server);
     if (!h.ok) {
-        // Include a truncated response snippet (when the extractor is the
+        return { state: 'pending', players: null, error: h.reason, restartedNow: false };
-        // failure) so the operator can spot which Komodo field to teach.
-        const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
-        return { state: 'pending', players: null, error: `unhealthy: ${h.reason}${snippet}`, restartedNow: false };
     }
-    const rcon = await rconStatusResult(server);
+    return { state: 'done', players: h.players, error: null, restartedNow: false };
-    if (!rcon.ok) {
-        return { state: 'pending', players: null, error: `rcon not answering: ${rcon.reason}`, restartedNow: false };
-    }
-    return { state: 'done', players: rcon.status.players, error: null, restartedNow: false };
 }

src/lib/server/pipeline/worker.ts

@@ -18,7 +18,15 @@ let started = false;
 let tickTimer: NodeJS.Timeout | null = null;
 let statusTimer: NodeJS.Timeout | null = null;
 
-async function tick(): Promise<void> {
+let tickInFlight: Promise<void> | null = null;
+
+function tick(): Promise<void> {
+    if (tickInFlight) return tickInFlight;
+    tickInFlight = runTick().finally(() => { tickInFlight = null; });
+    return tickInFlight;
+}
+
+async function runTick(): Promise<void> {
     try {
         await sweep();
     } catch (err) {

src/routes/deployments/+page.svelte

@@ -204,7 +204,7 @@
 <div class="page">
     <SectionHeader
         title="Deployments"
-        subtitle="Fleet-wide script deploys, drift reconciliation, and rollbacks."
+        subtitle="Fleet deployments, drift reconciliation, and rollbacks. Servers restart in stages as earlier servers recover."
     />
 
     {#if loading}

tests/unit/restart-rollout.test.ts

@@ -0,0 +1,333 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
+    komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
+}));
+vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
+    rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
+}));
+
+let fixture: Fixture;
+const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_theater_3'];
+const healthyRcon = {
+    ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
+    status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
+};
+
+beforeEach(async () => {
+    vi.resetModules();
+    vi.resetAllMocks();
+    vi.useFakeTimers({ toFake: ['Date'] });
+    vi.setSystemTime(new Date('2026-09-17T15:00:00Z'));
+    fixture = await newFixture(servers);
+    vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
+    vi.stubEnv('KOMODO_API_KEY', 'test');
+    vi.stubEnv('KOMODO_API_SECRET', 'test');
+    vi.stubEnv('RCON_PASSWORD', 'test');
+    vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
+    const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
+    await seedServerRegistry();
+    const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+    const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
+    vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
+    vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+    vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
+    vi.mocked(rconStatusResult).mockResolvedValue(healthyRcon);
+});
+
+afterEach(async () => {
+    await fixture.cleanup();
+    vi.unstubAllEnvs();
+    vi.useRealTimers();
+});
+
+function nextTick() { vi.setSystemTime(Date.now() + 15_000); }
+
+async function deployment(id = 'dep-1', targets = servers, action = 'promote') {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    db().prepare('INSERT INTO deployments (id, action, started_at) VALUES (?, ?, ?)')
+        .run(id, action, new Date().toISOString());
+    for (const server of targets) {
+        db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
+            VALUES (?, ?, 1, 'pending')`).run(id, server);
+    }
+}
+
+async function rows(id = 'dep-1') {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    return db().prepare('SELECT server, restart_state, restarted_at, error FROM deployment_servers WHERE deployment_id = ? ORDER BY server')
+        .all(id) as Array<{ server: string; restart_state: string; restarted_at: string | null; error: string | null }>;
+}
+
+async function beta(targets: string[], waitForHealth = true) {
+    const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
+    const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
+    const { script } = createScript({ name: 'test script', targetRelPath: 'rollout.gsc', content: Buffer.from('main(){}') });
+    return openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }],
+        servers: targets, waitForHealth, waitForMatch: false });
+}
+
+describe('rolling deployment restarts', () => {
+    it('restarts one server per recovery and finishes only after the last server is healthy', async () => {
+        await deployment();
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect((await sweep()).restarted).toBe(1);
+        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
+        expect((await rows()).map((row) => !!row.restarted_at)).toEqual([true, false, false]);
+        await sweep(); // Repeated kick can't bypass the grace period.
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+        nextTick();
+        expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
+        expect((await rows()).map((row) => row.restart_state)).toEqual(['done', 'pending', 'pending']);
+        nextTick();
+        expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
+        nextTick();
+        expect(await sweep()).toMatchObject({ restarted: 0, healthy: 1 });
+        expect((await rows()).every((row) => row.restart_state === 'done')).toBe(true);
+        expect(vi.mocked(komodoRestart).mock.calls).toEqual(servers.map((server) => [server]));
+        expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get('dep-1'))
+            .toMatchObject({ finished_at: expect.any(String) });
+    });
+
+    it('shares capacity across promote, rollback, and drift deployments in oldest-first order', async () => {
+        await deployment('z-oldest', [servers[2]], 'rollback');
+        nextTick();
+        await deployment('a-next', [servers[0]], 'drift');
+        nextTick();
+        await deployment('b-latest', [servers[1]], 'promote');
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        await sweep();
+        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
+        nextTick();
+        await sweep();
+        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]], [servers[0]]]);
+    });
+
+    it('skips occupied and unreachable servers without consuming capacity', async () => {
+        await deployment();
+        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+        vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0);
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        await sweep();
+        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
+    });
+
+    it('keeps the fleet queued while either container or RCON health is failing', async () => {
+        await deployment();
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
+        await sweep();
+        nextTick();
+        vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+        nextTick();
+        vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'starting' });
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+        nextTick();
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(2);
+    });
+
+    it('coalesces overlapping sweeps while the remote restart is unresolved', async () => {
+        await deployment();
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        let finish!: () => void;
+        const entered = new Promise<void>((resolve) => {
+            vi.mocked(komodoRestart).mockImplementationOnce(() => {
+                resolve();
+                return new Promise((done) => { finish = () => done({ ok: true }); });
+            });
+        });
+        const first = sweep();
+        await entered;
+        const second = sweep();
+        expect(second).toBe(first);
+        // Even a long-running restart request cannot be freed by a health poll.
+        nextTick();
+        const { refreshRestartRollout, rolloutWaitReason } = await import('../../src/lib/server/pipeline/restart-rollout.js');
+        await refreshRestartRollout();
+        expect(rolloutWaitReason(servers[1])).toMatch(/Queued/);
+        finish();
+        await Promise.all([first, second]);
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+    });
+
+    it('deduplicates simultaneous deployment rows for the same server', async () => {
+        await deployment('dep-1', [servers[0]]);
+        await deployment('dep-2', [servers[0], servers[1]]);
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+        expect((await rows('dep-1'))[0].restarted_at).toBeTruthy();
+        expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
+    });
+
+    it('does not count a later deployment as having been applied by an earlier restart', async () => {
+        await deployment('dep-1', [servers[0]]);
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        await sweep();
+        await deployment('dep-2', [servers[0]]);
+        nextTick();
+        await sweep();
+        expect((await rows('dep-1'))[0].restart_state).toBe('done');
+        expect((await rows('dep-2'))[0].restarted_at).toBeNull();
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(2);
+        expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
+    });
+
+    it('preserves occupied slots across app/module restarts', async () => {
+        await deployment();
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        await sweep();
+        const { closeDb } = await import('../../src/lib/server/db/index.js');
+        closeDb();
+        vi.resetModules();
+        const fresh = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        await fresh.sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+        nextTick();
+        await fresh.sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(2);
+    });
+
+    it('adopts pre-upgrade restarts instead of starting a second server', async () => {
+        await deployment();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?')
+            .run(new Date().toISOString(), servers[0]);
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        await sweep();
+        expect(komodoRestart).not.toHaveBeenCalled();
+    });
+
+    it('retains capacity after a timeout because the remote restart may have been accepted', async () => {
+        await deployment();
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' });
+        await sweep();
+        expect((await rows())[0].restart_state).toBe('failed');
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+        nextTick();
+        vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'restarting' });
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+    });
+
+    it('does not restart a server whose file deployment failed', async () => {
+        await deployment();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?")
+            .run(servers[0]);
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        await sweep();
+        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]);
+        expect((await rows())[0].error).toBe('write failed');
+    });
+
+    it('releases slots for definite local failures that could not have started a restart', async () => {
+        await deployment();
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' });
+        await sweep();
+        expect((await rows())[0].restart_state).toBe('failed');
+        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]], [servers[1]]]);
+    });
+});
+
+describe('beta and deployment coordination', () => {
+    it('limits beta restarts and releases capacity only after recovery', async () => {
+        await beta(servers);
+        const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        const first = betaTick();
+        expect(betaTick()).toBe(first);
+        await first;
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+        nextTick();
+        await betaTick();
+        expect(komodoRestart).toHaveBeenCalledTimes(2);
+    });
+
+    it('shares the same capacity when beta and deployment ticks run concurrently', async () => {
+        await beta([servers[1]]);
+        await deployment('dep-1', [servers[0]]);
+        const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        await Promise.all([sweep(), betaTick()]);
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+    });
+
+    it('retains beta recovery slots even when the beta skips health checks or is closed', async () => {
+        const { betaId } = await beta([servers[0]], false);
+        const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        await betaTick();
+        expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'ready' });
+        db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(betaId);
+        await deployment('dep-1', [servers[1]]);
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+        nextTick();
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(2);
+    });
+
+    it('checks player occupancy again when resuming a legacy restarting beta', async () => {
+        const { betaId } = await beta([servers[0]]);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?")
+            .run(new Date(Date.now() - 30_000).toISOString(), betaId);
+        const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+        vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
+        await betaTick();
+        expect(komodoRestart).not.toHaveBeenCalled();
+    });
+});
+
+describe('rollout configuration', () => {
+    it('honors a configured limit larger than one', async () => {
+        const { closeDb } = await import('../../src/lib/server/db/index.js');
+        closeDb();
+        vi.resetModules();
+        vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
+        await deployment();
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(2);
+        await sweep();
+        expect(komodoRestart).toHaveBeenCalledTimes(2);
+    });
+
+    it.each(['0', '-1', '1.5', 'abc'])('rejects invalid concurrency %s', async (value) => {
+        const { closeDb } = await import('../../src/lib/server/db/index.js');
+        closeDb();
+        vi.resetModules();
+        vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', value);
+        await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_RESTART_CONCURRENCY/);
+        vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
+        vi.resetModules();
+    });
+});