Add signed agent 1.9.2 release manifest

agent/ec_agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.9.1"
+VERSION = "1.9.2"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1504,7 +1504,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         plutonium_cache = self.plutonium_cache_inventory()
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
@@ -1889,7 +1889,7 @@ class Agent:
             managed_write(app, marker, raw, uid, gid)
             return True
 
-    def create(self, instance_id, generation, payload):
+    def create(self, instance_id, generation, payload, *, before=None):
         self.operation_phase = "checking-instance"
         self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
@@ -1930,7 +1930,7 @@ class Agent:
         args.append(image_id)
         self.check_creation_cancellation()
         self.operation_phase = "creating-container"
-        self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha})
+        self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha, **({"before": before} if before is not None else {})})
         self.docker.call(*args, timeout=120)
         self.check_creation_cancellation()
         self.operation_phase = "starting-container"
@@ -2126,7 +2126,11 @@ class Agent:
                 self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
             if "autoscaling" in payload:
                 self.authorize_autoscaling_removal(instance_id, generation, payload)
-            self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
+            if kind == "instance.rebuild" and payload.get("stageId"):
+                before = self.rebuild_before(instance_id, generation, payload["stageId"], spec)
+                if not override_safety:
+                    self.safe_to_disrupt(instance_id, generation)
+            self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})})
             if payload.get("stageId") and kind == "instance.restart":
                 before = self.activate(instance_id, generation, payload["stageId"], override_safety=override_safety)
             if kind in ("instance.remove", "instance.rebuild"):
@@ -2135,12 +2139,12 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 self.settle_bank(instance_id, generation)
                 if kind == "instance.rebuild":
-                    result = self.create(instance_id, next_generation, spec)
+                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None)
                     self.operation_phase = "removing-old-instance"
                     old = self.instance_dir(instance_id, generation)
                     if old.exists():
                         shutil.rmtree(old, ignore_errors=False)
-                    return result
+                    return {**result, "before": before}
                 directory = self.instance_dir(instance_id, generation)
                 if directory.exists():
                     shutil.rmtree(directory)
@@ -2235,6 +2239,29 @@ class Agent:
         atomic_json(marker, manifest)
         return {"stageId": stage_id, "files": len(planned), "state": "staged"}
 
+    def rebuild_before(self, instance_id, generation, stage_id, spec):
+        """Validate requested replacement bytes and snapshot the old generation before teardown."""
+        directory = self.instance_dir(instance_id, generation)
+        marker = directory / "stages" / identifier(stage_id) / "manifest.json"
+        saved = json.loads(marker.read_text())
+        if saved["state"] != "staged":
+            raise Rejected("Rebuild requires an unactivated file stage")
+        replacement = {item["path"]: item for item in self.initial_files(spec)}
+        before = []
+        for item in saved["files"]:
+            target = replacement.get(item["path"])
+            if item["delete"]:
+                if target is not None:
+                    raise Rejected("Rebuild includes a file requested for removal")
+            elif target is None or digest(self.decode_file(target)) != item["sha256"]:
+                raise Rejected("Rebuild does not match staged file content")
+            content = managed_read(directory / "app", item["path"])
+            actual_sha = None if content is None else digest(content)
+            if item.get("checkBefore") and actual_sha != item.get("expectedBeforeSha"):
+                raise Rejected("Target changed after campaign review")
+            before.append({"id": item.get("id"), "path": item["path"], "contentBase64": None if content is None else base64.b64encode(content).decode()})
+        return before
+
     def activate(self, instance_id, generation, stage_id, *, override_safety=False):
         directory = self.instance_dir(instance_id, generation)
         stage = directory / "stages" / identifier(stage_id)
@@ -2966,6 +2993,12 @@ class Agent:
             health = self.inspect(instance_id, observed_generation)
             before = []
             if payload.get("stageId"):
+                if kind == "instance.rebuild":
+                    # The creating checkpoint retains snapshots after old storage is deleted.
+                    if not isinstance(checkpoint.get("before"), list):
+                        return row
+                    before = checkpoint["before"]
+                else:
                     stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json"
                     saved = json.loads(stage.read_text())
                     if saved["state"] != "activated":

agent/tests/test_agent.py

@@ -1821,6 +1821,66 @@ class AgentTests(unittest.TestCase):
             self.assertTrue(path.is_symlink())
             request.assert_not_called()
 
+    def staged_rebuild(self, deleting=False):
+        spec = self.create()
+        self.agent.rcon = lambda *_: {"ok": True, "players": 0}
+        file = {"id": 9, "path": "configs/server.cfg", "contentBase64": None if deleting else base64.b64encode(b"replacement").decode(), "checkBefore": True, "expectedBeforeSha": module.digest(b"original")}
+        staged = self.agent.execute(self.operation("files.stage", {"stageId": "campaign", "files": [file]}, "stage"))
+        self.assertEqual(staged["state"], "completed")
+        spec["files"] = [] if deleting else [file]
+        return self.operation("instance.rebuild", {"spec": spec, "stageId": "campaign", "nextGeneration": 2}, "rebuild")
+
+    def test_file_rebuild_replaces_generation_and_retains_before_images(self):
+        operation = self.staged_rebuild()
+        result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed", result)
+        receipt = json.loads(result["result"])
+        self.assertEqual(receipt["before"], [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"original").decode()}])
+        self.assertEqual(receipt["generation"], 2)
+        self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists())
+        self.assertEqual((self.agent.instance_dir(INSTANCE, 2) / "app/configs/server.cfg").read_bytes(), b"replacement")
+        self.assertIn("instance.rebuild-files", self.agent.inventory()["capabilities"])
+        calls = list(self.docker.calls)
+        self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertEqual(self.docker.calls, calls)
+
+    def test_file_rebuild_removes_requested_files(self):
+        operation = self.staged_rebuild(deleting=True)
+        result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed", result)
+        self.assertFalse((self.agent.instance_dir(INSTANCE, 2) / "app/configs/server.cfg").exists())
+        self.assertEqual(len(json.loads(result["result"])["before"]), 1)
+
+    def test_file_rebuild_rejects_changed_targets_and_mismatched_replacement(self):
+        operation = self.staged_rebuild()
+        target = self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg"
+        target.write_bytes(b"changed")
+        self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        target.write_bytes(b"original")
+        operation["id"] = "mismatched"
+        operation["payload"]["spec"]["files"] = []
+        self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
+
+    def test_file_rebuild_rechecks_players_after_snapshot_preparation(self):
+        operation = self.staged_rebuild()
+        with patch.object(self.agent, "rcon", side_effect=[{"ok": True, "players": 0}, {"ok": True, "players": 1}]):
+            self.assertEqual(self.agent.execute(operation)["state"], "accepted")
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
+
+    def test_file_rebuild_reconciles_snapshots_after_old_storage_was_removed(self):
+        operation = self.staged_rebuild()
+        self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.agent.journal.update(operation["id"], "unknown", error="Lost completion")
+        calls = list(self.docker.calls)
+        result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(json.loads(result["result"])["before"][0]["contentBase64"], base64.b64encode(b"original").decode())
+        self.assertEqual(self.docker.calls, calls)
+
     def test_rebuild_moves_generation_and_rejects_old_commands(self):
         payload = self.create()
         self.agent.rcon = lambda *_: {"ok": True, "players": 0}

docs/guides/beta-test-plans.md

@@ -29,8 +29,8 @@ servers. Servers held by another active beta or with pending deployment work
 are excluded, as are instances on unavailable, draining, or disabled hosts.
 Unknown or stale player counts are not treated as empty. If the
 requested counts cannot be met, including coverage of every selected map-scoped
-file, the beta does not start. Deployment and restarts still wait for confirmed
-empty lobbies and restart capacity.
+file, the beta does not start. File deployment rebuilds servers after waiting
+for confirmed empty lobbies and rebuild capacity.
 
 Open **Advanced options → Choose specific servers** for manual selection. The
 picker includes servers matching any file in the bundle; each file must have a
@@ -349,7 +349,7 @@ Health monitoring continues after a pass until promotion.
 
 For failures, select hold as failed, pause for investigation, or automatic
 rollback. Rollback queues restoration only for files touched by the beta.
-Restoration and restart both wait for an empty server and restart capacity.
+Restoration rebuilds the server after waiting for an empty lobby and rebuild capacity.
 The manual option holds those changes in a rollback campaign until you select
 **Resume when empty**. Promotion is always manual.
 
@@ -361,9 +361,9 @@ Review the promotion preview before confirming. API clients fetch it with
 409 and requires a fresh preview.
 
 Confirming promotion creates a campaign without reading the existing files on
-additional servers. Each server waits for an empty lobby and restart capacity,
-then captures rollback snapshots immediately before applying its queued files
-and restarting. Campaign history records snapshot capture, and file comparisons
+additional servers. Each server waits for an empty lobby and rebuild capacity,
+then captures rollback snapshots before rebuilding with its queued files.
+Campaign history records snapshot capture, and file comparisons
 become available once those snapshots are confirmed. Tested servers retain their
 original pre-beta snapshots; their composed configuration is still checked at
 promotion to determine whether another activation is needed.
@@ -393,9 +393,9 @@ binary and large files show their checksums. Files must actually change.
 
 The beta keeps its ID, servers, scope and test policy. Each revision creates new
 file versions, clears current test progress and checklist sign-offs, repeats the
-configured waves, and starts a fresh testing window at submission. Files install
-and servers restart through the normal empty-server and capacity gates. Promotion
-requires the new revision to pass. Pause and retry retain the current revision's
+configured waves, and starts a fresh testing window at submission. Servers rebuild
+with the requested files through the normal empty-server and capacity gates.
+Promotion requires the new revision to pass. Pause and retry retain the current revision's
 deadline; they do not create a revision.
 
 Revision history retains downloadable bundle files and previous test results,
@@ -407,7 +407,7 @@ new revision is waiting to deploy. Files originally absent are removed.
 Updates are available until promotion or rollback, including paused, failed and
 expired tests. An activation already in progress must settle first; interrupted
 file writes and deleted participants must be resolved before an update. Pending,
-unsubmitted restarts are superseded. Adding/removing files, changing targets or
+unsubmitted activations are superseded. Adding/removing files, changing targets or
 changing the policy still requires a new beta.
 
 ## API and storage

docs/guides/provisioning.md

@@ -150,14 +150,19 @@ and post-action health checks still apply.
 
 ## Campaigns, betas and maintenance
 
-Campaigns, beta runs, and maintenance submit durable restart requests to
+Campaign and beta file changes submit durable rebuild requests to
 Provisioning and observe the resulting operation by ID. Their pages link to that
 operation, and its history links back to the requesting work. Provisioning owns
-the empty-server check, conflict handling, restart capacity, remote command, and
+the empty-server check, conflict handling, capacity limits, remote command, and
 health verification. The callers retain file deployment, beta evidence, and
 maintenance outcome verification. Jobs remain visible while waiting for players,
 other work, or integration configuration.
 
+File rebuilds require host agent 1.9.2 or later. They capture rollback snapshots
+before replacing the instance, and install the requested files over the promoted
+setup. Queued file activations wait for the agent update when necessary.
+Maintenance jobs and beta recovery cycles without file changes still use restarts.
+
 [Maintenance](maintenance.md) also submits recovery operations and links each attempt
 to its incident. A failed Maintenance restart can hand its reserved slot to a rebuild
 after five minutes; unrelated recovery slots remain held. Drift campaigns follow

src/lib/server/fleet/servers.ts

@@ -190,14 +190,17 @@ export function listServers(): ServerRow[] {
 }
 
 /** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */
-export function activeBetaRemovalBlocker(server: string): string | null {
+export function activeBetaRemovalBlocker(server: string, activationId?: string): string | null {
     const betas = db()
         .prepare(
             `SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
         WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')
+        AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id=?
+            AND s.kind='beta' AND s.source_id=CAST(b.id AS TEXT) AND s.server=bs.server
+            AND bs.provisioning_id=s.operation_id)
         ORDER BY b.id`
         )
-        .all(server) as { id: number }[];
+        .all(server, activationId ?? null) as { id: number }[];
     if (!betas.length) return null;
     return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map((beta) => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`;
 }

src/lib/server/pipeline/beta-plan-runner.ts

@@ -56,11 +56,11 @@ export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void
     if (!result.deploymentId)
         activity(id, 'Cancelled queued changes; no deployed files need restoration');
     else if (plan.rollbackRestart === 'when-empty')
-        activity(id, 'Queued file restoration and rolling restarts; waiting for empty servers');
+        activity(id, 'Queued file restoration and rolling rebuilds; waiting for empty servers');
     else
         activity(
             id,
-            'File restoration is held; resume the rollback campaign to apply it with a safe restart'
+            'File restoration is held; resume the rollback campaign to apply it with a safe rebuild'
         );
     if (result.warnings.length) activity(id, result.warnings.join('; '));
 }

src/lib/server/provisioning/activation-files.ts

@@ -0,0 +1,44 @@
+import { db } from '../db/index.js';
+import { blobSize, getFileBlob } from '../db/file-blobs.js';
+import type { LaunchRecipePlan } from '../fleet/launch-recipes.js';
+import type { StagedFileChange } from '../fleet/staged-files.js';
+
+/** Build the replacement with the exact requested additions, updates and removals. */
+export function applyActivationFiles(
+    recipe: LaunchRecipePlan,
+    operationId: string,
+    includeContent = false
+): void {
+    const changes = db()
+        .prepare(
+            `SELECT f.*, ${blobSize('f.content_hash')} AS size, v.version FROM staged_file_changes f
+        JOIN provisioning_sources s ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
+        LEFT JOIN script_versions v ON v.id=f.script_version_id
+        WHERE s.operation_id=? AND f.state='pending' ORDER BY f.id`
+        )
+        .all(operationId) as (StagedFileChange & {
+        content_hash: string | null;
+        size: number;
+        version: number | null;
+    })[];
+    for (const change of changes) {
+        if (change.kind !== 'script' && change.kind !== 'config' && change.kind !== 'plugin')
+            throw new Error('File activation requires a runtime file kind.');
+        recipe.files = recipe.files.filter(
+            (file) => file.kind !== change.kind || file.path !== change.rel_path
+        );
+        if (!change.content_hash) continue;
+        recipe.files.push({
+            kind: change.kind,
+            name: change.rel_path,
+            path: change.rel_path,
+            versionId: change.script_version_id,
+            version: change.version,
+            sha256: change.content_hash,
+            size: change.size,
+            ...(includeContent
+                ? { content: getFileBlob(change.content_hash)!.toString('base64') }
+                : {})
+        });
+    }
+}

src/lib/server/provisioning/launch-preparation.ts

@@ -19,6 +19,8 @@ export interface NativeLaunchInput {
     unhealthyRecovery?: boolean;
     /** Rebalance replays recorded deployed bytes instead of current promoted files. */
     frozenRecipeId?: string;
+    /** Pending beta/campaign files override promoted launch files. */
+    activationId?: string;
 }
 
 export interface PreparedNativeLaunch {

src/lib/server/provisioning/launch-worker.ts

@@ -1,5 +1,6 @@
 import { parentPort } from 'node:worker_threads';
 import { posix } from 'node:path';
+import { applyActivationFiles } from './activation-files.js';
 import { db } from '../db/index.js';
 import { relDestForKind } from '../config.js';
 import {
@@ -32,15 +33,15 @@ parentPort!.on(
             const input = message.input;
             // All file versions and composed configuration belong to one snapshot.
             const { recipe, password } = db().transaction(() => {
-                if (!input.frozenRecipeId)
-                    return {
-                        recipe: {
+                if (!input.frozenRecipeId) {
+                    const recipe = {
                         ...captureRecipe(input.game, input.mapSlug, input.rconPassword),
                         native: input.native,
                         image: input.instance.image
-                        },
-                        password: input.rconPassword
                     };
+                    if (input.activationId) applyActivationFiles(recipe, input.activationId, true);
+                    return { recipe, password: input.rconPassword };
+                }
                 const stored = recipeForDeployment(input.frozenRecipeId)?.recipe;
                 const migration = db()
                     .prepare(
@@ -85,6 +86,7 @@ parentPort!.on(
             )
                 throw new Error('electric_cherry_bank.cfg is reserved for instance credentials.');
             const payload = {
+                ...(input.activationId ? { stageId: input.activationId } : {}),
                 name: instance.server,
                 game: recipe.game,
                 libraryId: instance.library_id,

src/lib/server/provisioning/native.ts

@@ -18,6 +18,7 @@ import { db } from '../db/index.js';
 import { unsettledCancelledCommand } from './cancellation.js';
 import { config, relDestForKind, serverDir } from '../config.js';
 import {
+    runtimePasswordForServer,
     nativeInstance,
     nativeRequest,
     unplacedNativeCreation,
@@ -55,6 +56,7 @@ import {
     availableServerKeyCount
 } from '../fleet/server-keys.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
+import { applyActivationFiles } from './activation-files.js';
 import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
 import {
     beginDeployment,
@@ -916,7 +918,10 @@ function queueUnhealthyRecovery(job: Job, instance: NativeInstance): void {
         }
         let payload: Record<string, unknown> = { unhealthyRecovery: true };
         if (step === 'rebuild') {
-            const beta = activeBetaRemovalBlocker(job.server);
+            const beta = activeBetaRemovalBlocker(
+                job.server,
+                job.action === 'rebuild' ? job.id : undefined
+            );
             if (beta) {
                 hold(job, beta);
                 return;
@@ -952,6 +957,7 @@ function queueUnhealthyRecovery(job: Job, instance: NativeInstance): void {
                 unhealthyRecovery: true
             };
             delete payload.safetyOverride;
+            delete payload.stageId;
         }
         const operation = queueHostOperationSerialized(
             instance.host_id,
@@ -1701,6 +1707,17 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             hold(job, 'Waiting for current health evidence before completing verification.');
         return;
     }
+    // A queued activation from a previous controller release may still say restart.
+    // Upgrade only intent that has never submitted a host command.
+    if (
+        job.action === 'restart' &&
+        job.state === 'waiting' &&
+        !job.submitted_at &&
+        filesFor(job).length
+    ) {
+        db().prepare("UPDATE server_deployments SET action='rebuild' WHERE id=?").run(job.id);
+        job.action = 'rebuild';
+    }
     const reason = eligibility(job);
     if (reason) {
         hold(job, reason);
@@ -1712,13 +1729,58 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         return;
     }
     if (job.action === 'destroy' || job.action === 'rebuild') {
-        const beta = activeBetaRemovalBlocker(job.server);
+        const beta = activeBetaRemovalBlocker(
+            job.server,
+            job.action === 'rebuild' ? job.id : undefined
+        );
         if (beta) {
             hold(job, beta);
             return;
         }
     }
     const files = job.stage_operation_id ? storedFiles(job) : filesFor(job);
+    if (
+        job.action === 'rebuild' &&
+        files.length &&
+        !host.inventory.capabilities?.includes('instance.rebuild-files')
+    ) {
+        hold(job, 'Update the host agent to support beta and campaign file rebuilds.');
+        return;
+    }
+    if (job.action === 'rebuild' && !recipeSummaryForDeployment(job.id)) {
+        const server = getServer(job.server);
+        if (!server?.map_slug) throw new Error('A saved map is required before rebuilding.');
+        const recipe: NativeRecipe = {
+            ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug),
+            image: instance.image,
+            native: {
+                hostId: instance.host_id,
+                libraryId: instance.library_id,
+                image: instance.image,
+                bindAddress: instance.bind_address
+            }
+        };
+        const stored = recipeStorage(recipe);
+        db().transaction(() => {
+            db()
+                .prepare(
+                    'INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)'
+                )
+                .run(
+                    job.id,
+                    job.server,
+                    instance!.port,
+                    'zombie',
+                    recipe.mapSlug,
+                    recipe.expectedMap
+                );
+            db()
+                .prepare(
+                    'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)'
+                )
+                .run(job.id, stored.recipe, stored.recipeSummary);
+        })();
+    }
     if (files.length && !job.stage_operation_id) {
         db().transaction(() => {
             const stage = queueHostOperationSerialized(
@@ -1838,7 +1900,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             'instance.inspect'
         );
         if (!stillActive(job)) return false;
-        if (job.action === 'rebuild' && health.state === 'unhealthy') {
+        if (job.action === 'rebuild' && !files.length && health.state === 'unhealthy') {
             if (
                 !getHost(instance.host_id)?.inventory.capabilities?.includes(
                     unhealthyRecoveryCapability
@@ -1929,7 +1991,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 phase: 'prepare',
                 checkpoint: { key: 'startup-queue', label: 'Startup capacity available' }
             });
-            preparedPassword = config.rcon.password || randomBytes(32).toString('hex');
+            preparedPassword =
+                (files.length ? runtimePasswordForServer(job.server) : undefined) ||
+                config.rcon.password ||
+                randomBytes(32).toString('hex');
             prepared = await prepareNativeLaunch({
                 game: prior.game,
                 mapSlug: prior.mapSlug,
@@ -1940,9 +2005,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 bankToken: bankCredential(instance.id),
                 rebuild: job.action === 'rebuild' && !migration,
                 safetyOverride: overrideSafety,
-                unhealthyRecovery: host.inventory.capabilities?.includes(
-                    unhealthyRecoveryCapability
-                ),
+                unhealthyRecovery:
+                    !files.length &&
+                    host.inventory.capabilities?.includes(unhealthyRecoveryCapability),
+                activationId: files.length ? job.id : undefined,
                 frozenRecipeId
             });
             if (job.action === 'rebuild' && !migration && !(await checkOccupancy())) return;
@@ -1977,7 +2043,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 return;
             }
             if (job.action === 'destroy' || job.action === 'rebuild') {
-                const beta = activeBetaRemovalBlocker(job.server);
+                const beta = activeBetaRemovalBlocker(
+                    job.server,
+                    job.action === 'rebuild' ? job.id : undefined
+                );
                 if (beta) {
                     hold(job, beta);
                     return;
@@ -2032,6 +2101,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                               prepared.recipe.mapSlug,
                               preparedPassword
                           );
+                if (files.length) applyActivationFiles(expected, job.id);
                 if (
                     getServerKey(job.server) !== preparedKey ||
                     !sameLaunchFiles(prepared.recipe, expected)
@@ -2119,7 +2189,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                             recipe.image,
                             instance!.port
                         );
-                for (const file of recipe.files)
+                for (const file of recipe.files.filter(
+                    (file) =>
+                        !files.some(
+                            (change) =>
+                                change.path ===
+                                posix.join(relDestForKind(file.kind, recipe.game), file.path)
+                        )
+                ))
                     recordScript(
                         job.server,
                         file.path,

src/lib/server/provisioning/requests.ts

@@ -43,7 +43,7 @@ const sourceTables = {
     maintenance: { table: 'maintenance_jobs', key: 'id' }
 } as const;
 
-/** Submit once per caller attempt; a lost response or terminal failure never replays a restart. */
+/** File activations rebuild; maintenance and beta recovery cycles without files restart. */
 export function requestRestart(server: string, source: RestartSource): LifecycleOperation {
     return db().transaction(() => {
         const { table, key } = sourceTables[source.kind];
@@ -54,6 +54,13 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
             .get(source.id, server) as
             { provisioning_id: string | null; restarted_at: string | null } | undefined;
         if (!row) throw new Error('Restart source no longer exists');
+        const activatesFiles =
+            source.kind !== 'maintenance' &&
+            !!db()
+                .prepare(
+                    "SELECT 1 FROM staged_file_changes WHERE source_kind=? AND source_id=? AND server=? AND state='pending' LIMIT 1"
+                )
+                .get(source.kind, source.id, server);
         if (row.provisioning_id) {
             const operation = provisioningOperation(row.provisioning_id);
             if (!operation)
@@ -62,6 +69,18 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
                 adoptSubmittedRestart(operation.id, server, row.restarted_at);
                 return provisioningOperation(operation.id)!;
             }
+            if (
+                activatesFiles &&
+                !row.restarted_at &&
+                !operation.submitted_at &&
+                operation.state === 'waiting' &&
+                operation.action === 'restart'
+            ) {
+                db()
+                    .prepare("UPDATE server_deployments SET action='rebuild' WHERE id=?")
+                    .run(operation.id);
+                return provisioningOperation(operation.id)!;
+            }
             return operation;
         }
         let id: string | undefined;
@@ -84,7 +103,11 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
                     .run(row.restarted_at, id);
         }
         if (!id) {
-            id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
+            id = beginDeployment(
+                server,
+                activatesFiles && !row.restarted_at ? 'rebuild' : 'restart',
+                `${source.kind}:${source.id}`
+            );
             db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
             deploymentStage(id, 'Waiting to verify the server is empty', {
                 phase: 'eligibility',

src/routes/betas/[id]/+page.svelte

@@ -297,7 +297,7 @@
         const restarts =
             test?.plan.rollbackRestart === 'manual'
                 ? 'Resume the rollback campaign when ready; files stay unchanged until safety checks pass.'
-                : 'Files are restored just before restarting, after safety checks pass.';
+                : 'Servers rebuild with the original files after safety checks pass.';
         if (
             !(await confirmAction({
                 title: reject ? 'Reject beta?' : 'Roll back beta?',
@@ -557,7 +557,7 @@
                                         ><td colspan="4">
                                             <p class="muted">
                                                 Files {server.deployed_at ? 'installed' : 'pending'} ·
-                                                Restart {server.restarted_at
+                                                Activation {server.restarted_at
                                                     ? 'submitted'
                                                     : 'pending'}{#if beta.wait_for_health}
                                                     · Health {server.healthy_at
@@ -734,10 +734,10 @@
         </p>
         <p>
             Confirming creates a campaign and queues the file changes. Each server waits until empty
-            and restart capacity is available, then captures rollback snapshots, installs its files,
-            and restarts. Tested servers keep their pre-beta snapshots and only need another
-            activation if their configuration changed or earlier queued work could replace the
-            tested files. Eligibility follows each file's current scope.
+            and rebuild capacity is available, then captures rollback snapshots and rebuilds with
+            its files. Tested servers keep their pre-beta snapshots and only need another activation
+            if their configuration changed or earlier queued work could replace the tested files.
+            Eligibility follows each file's current scope.
         </p>
         <p class="muted">
             Reviewed {$timeFormat.dateTime(preview.createdAt)}. If targets, versions, or composed
@@ -756,7 +756,7 @@
                                 <ServerIdentity {name} /><span
                                     >{preview.servers.find((server) => server.name === name)?.tested
                                         ? 'Already tested'
-                                        : 'Additional target · restart when empty'}</span
+                                        : 'Additional target · rebuild when empty'}</span
                                 >
                             </li>{/each}
                     </ul>

src/routes/betas/new/+page.svelte

@@ -989,12 +989,12 @@
                               ? 'Failed checks pause testing.'
                               : 'Failed checks hold the beta for review.'}
                         {testPlan.rollbackRestart === 'when-empty'
-                            ? 'Rollback restarts wait for empty servers.'
-                            : 'Rollback restores files; restart servers manually.'}
+                            ? 'Rollback rebuilds wait for empty servers.'
+                            : 'Rollback is held until you resume its campaign.'}
                     </p>
                     <p>
                         Starting installs each selected version only on matching test servers and
-                        restarts them when empty. Waiting, pausing, and retrying use the original
+                        rebuilds them when empty. Waiting, pausing, and retrying use the original
                         testing window. Promotion is a separate reviewed action after testing.
                     </p>
                 </section>{/if}

src/routes/campaigns/[id]/+page.svelte

@@ -349,7 +349,7 @@
                     <p>
                         {detail.watchdogIncidentId
                             ? 'Maintenance rebuilds instance files with the latest promoted setup after checking that the server is safe to update.'
-                            : 'Changes wait for an empty server and restart capacity. Rollback snapshots are captured before applying files, then servers restart and verify health. Tested servers retain their pre-beta snapshots.'}
+                            : 'Changes wait for an empty server and rebuild capacity. Rollback snapshots are captured before servers rebuild with the requested files and verify health. Tested servers retain their pre-beta snapshots.'}
                     </p>
                 </div>
             </details>
@@ -495,7 +495,7 @@
                                                     <dd><RelativeTime value={s.checked_at} /></dd>
                                                 </div>{/if}
                                             {#if s.restarted_at}<div>
-                                                    <dt>Restart requested</dt>
+                                                    <dt>Rebuild requested</dt>
                                                     <dd><RelativeTime value={s.restarted_at} /></dd>
                                                 </div>{/if}
                                             {#if s.server_deleted_at}<div>

static/agent/ec-agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.9.1"
+VERSION = "1.9.2"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1504,7 +1504,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         plutonium_cache = self.plutonium_cache_inventory()
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
@@ -1889,7 +1889,7 @@ class Agent:
             managed_write(app, marker, raw, uid, gid)
             return True
 
-    def create(self, instance_id, generation, payload):
+    def create(self, instance_id, generation, payload, *, before=None):
         self.operation_phase = "checking-instance"
         self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
@@ -1930,7 +1930,7 @@ class Agent:
         args.append(image_id)
         self.check_creation_cancellation()
         self.operation_phase = "creating-container"
-        self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha})
+        self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha, **({"before": before} if before is not None else {})})
         self.docker.call(*args, timeout=120)
         self.check_creation_cancellation()
         self.operation_phase = "starting-container"
@@ -2126,7 +2126,11 @@ class Agent:
                 self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
             if "autoscaling" in payload:
                 self.authorize_autoscaling_removal(instance_id, generation, payload)
-            self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
+            if kind == "instance.rebuild" and payload.get("stageId"):
+                before = self.rebuild_before(instance_id, generation, payload["stageId"], spec)
+                if not override_safety:
+                    self.safe_to_disrupt(instance_id, generation)
+            self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})})
             if payload.get("stageId") and kind == "instance.restart":
                 before = self.activate(instance_id, generation, payload["stageId"], override_safety=override_safety)
             if kind in ("instance.remove", "instance.rebuild"):
@@ -2135,12 +2139,12 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 self.settle_bank(instance_id, generation)
                 if kind == "instance.rebuild":
-                    result = self.create(instance_id, next_generation, spec)
+                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None)
                     self.operation_phase = "removing-old-instance"
                     old = self.instance_dir(instance_id, generation)
                     if old.exists():
                         shutil.rmtree(old, ignore_errors=False)
-                    return result
+                    return {**result, "before": before}
                 directory = self.instance_dir(instance_id, generation)
                 if directory.exists():
                     shutil.rmtree(directory)
@@ -2235,6 +2239,29 @@ class Agent:
         atomic_json(marker, manifest)
         return {"stageId": stage_id, "files": len(planned), "state": "staged"}
 
+    def rebuild_before(self, instance_id, generation, stage_id, spec):
+        """Validate requested replacement bytes and snapshot the old generation before teardown."""
+        directory = self.instance_dir(instance_id, generation)
+        marker = directory / "stages" / identifier(stage_id) / "manifest.json"
+        saved = json.loads(marker.read_text())
+        if saved["state"] != "staged":
+            raise Rejected("Rebuild requires an unactivated file stage")
+        replacement = {item["path"]: item for item in self.initial_files(spec)}
+        before = []
+        for item in saved["files"]:
+            target = replacement.get(item["path"])
+            if item["delete"]:
+                if target is not None:
+                    raise Rejected("Rebuild includes a file requested for removal")
+            elif target is None or digest(self.decode_file(target)) != item["sha256"]:
+                raise Rejected("Rebuild does not match staged file content")
+            content = managed_read(directory / "app", item["path"])
+            actual_sha = None if content is None else digest(content)
+            if item.get("checkBefore") and actual_sha != item.get("expectedBeforeSha"):
+                raise Rejected("Target changed after campaign review")
+            before.append({"id": item.get("id"), "path": item["path"], "contentBase64": None if content is None else base64.b64encode(content).decode()})
+        return before
+
     def activate(self, instance_id, generation, stage_id, *, override_safety=False):
         directory = self.instance_dir(instance_id, generation)
         stage = directory / "stages" / identifier(stage_id)
@@ -2966,6 +2993,12 @@ class Agent:
             health = self.inspect(instance_id, observed_generation)
             before = []
             if payload.get("stageId"):
+                if kind == "instance.rebuild":
+                    # The creating checkpoint retains snapshots after old storage is deleted.
+                    if not isinstance(checkpoint.get("before"), list):
+                        return row
+                    before = checkpoint["before"]
+                else:
                     stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json"
                     saved = json.loads(stage.read_text())
                     if saved["state"] != "activated":

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.9.1",
+  "version": "1.9.2",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "c3c9fc251c93802e8016b67dc3fba409df00de6681751b9ba231b80d702c7333",
+  "sha256": "acff8fdbf75652821638a9c90ee9cc4ef5254bc2f19fabc436b9135f93307a5a",
   "updaterProtocolVersion": 2,
   "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb",
   "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231"

static/agent/manifest.json.sig

@@ -1,8 +1,8 @@
 -----BEGIN PGP SIGNATURE-----
 
-iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCas2WORsUgAAAAAAEAA5t
-YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfrMYQEA3BJSdUa3uJW7Jz8J5+Ia
-v+zC8uuzBB7nXgW4gtbHguYA/i0M3faiIr2eQ4Og+fKqh2TVyAC2BhMbwBPbPaft
-JwIA
-=x+Vg
+iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCas2drBsUgAAAAAAEAA5t
+YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfqXUwD/UtXrU4iXCqcAGzYY+ETl
+JyvOrDoi5txZ3qz4pkwsZZIBAO3nz5BzaX4k3GfvKOYFsBJsfHDa76WeWwuC1h7m
+zdoN
+=mSzg
 -----END PGP SIGNATURE-----

tests/unit/beta-plans.test.ts

@@ -22,7 +22,7 @@ vi.mock('../../src/lib/server/provisioning/worker.js', () => ({
             await import('../../src/lib/server/fleet/status-evidence.js');
         const jobs = db()
             .prepare(
-                "SELECT id,server,state FROM server_deployments WHERE action='restart' AND state IN ('waiting','awaiting-health')"
+                "SELECT id,server,state FROM server_deployments WHERE action IN ('restart','rebuild') AND state IN ('waiting','awaiting-health')"
             )
             .all() as { id: string; server: string; state: string }[];
         for (const job of jobs) {
@@ -243,7 +243,7 @@ describe('durable beta test plans', () => {
         expect(getScript(file.script_id)?.current_version_id).toBe(replacement);
     });
 
-    it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => {
+    it('waits for occupied telemetry to clear before rebuilding a beta despite an empty RCON response', async () => {
         const id = await openPlan(defaultPlan());
         const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
         await postTelemetry(gameEvent('heartbeat', 4, 2));
@@ -437,7 +437,7 @@ describe('durable beta test plans', () => {
         await expect(promoteBeta(id)).rejects.toThrow();
     });
 
-    it('automatically rolls back only the deployed wave and queues safe rollback restarts', async () => {
+    it('automatically rolls back only the deployed wave and queues safe rollback rebuilds', async () => {
         await writeFile(scriptPath(names[0]), 'original');
         const id = await openPlan(
             {

tests/unit/beta-staging.test.ts

@@ -290,7 +290,13 @@ describe('staged beta files', () => {
             if (blocked)
                 await expect(request).rejects.toMatchObject({
                     kind: 'conflict',
-                    details: [expect.objectContaining({ deploymentId: promotion.deploymentId })]
+                    ...(operationState === 'waiting'
+                        ? { message: expect.stringContaining('rebuild') }
+                        : {
+                              details: [
+                                  expect.objectContaining({ deploymentId: promotion.deploymentId })
+                              ]
+                          })
                 });
             else await expect(request).resolves.toMatchObject({ servers: [servers[1]] });
         }

tests/unit/campaign-resume.test.ts

@@ -128,6 +128,29 @@ describe('campaign resume API', () => {
         }
     );
 
+    it('upgrades queued file restarts without replaying submitted operations', async () => {
+        await heldRollback('pending');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
+        const job = requestRestart(server, { kind: 'campaign', id: campaign });
+        expect(job.action).toBe('rebuild');
+        db().prepare("UPDATE server_deployments SET action='restart' WHERE id=?").run(job.id);
+        expect(requestRestart(server, { kind: 'campaign', id: campaign })).toMatchObject({
+            id: job.id,
+            action: 'rebuild'
+        });
+        db()
+            .prepare(
+                "UPDATE server_deployments SET action='restart',state='running',submitted_at=? WHERE id=?"
+            )
+            .run(new Date().toISOString(), job.id);
+        expect(requestRestart(server, { kind: 'campaign', id: campaign })).toMatchObject({
+            id: job.id,
+            action: 'restart',
+            state: 'running'
+        });
+    });
+
     it('retains an existing waiting operation when its held campaign resumes', async () => {
         await heldRollback();
         const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');

tests/unit/native-provisioning.test.ts

@@ -39,7 +39,7 @@ const inventory = () => ({
     memoryTotal: 64 * 1024 ** 3,
     memoryAvailable: 60 * 1024 ** 3,
     protocolVersion: 1,
-    capabilities: ['instance.unhealthy-recovery'],
+    capabilities: ['instance.unhealthy-recovery', 'instance.rebuild-files'],
     dockerRunning: true,
     directoriesWritable: true,
     storageAvailable: 100_000_000_000,
@@ -1725,6 +1725,96 @@ describe('native instance provisioning', () => {
         const { getHost } = await import('../../src/lib/server/hosts/index.js');
         expect(getHost(hostId)?.mode).toBe('active');
     });
+    it('builds campaign replacements with requested plugins, configuration, additions and removals', async () => {
+        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        for (const path of ['replace.dll', 'remove.dll']) {
+            const file = createScript({
+                name: path,
+                kind: 'plugin',
+                targetRelPath: path,
+                scopeType: 'game',
+                scopeValue: 't5',
+                content: Buffer.from('promoted plugin')
+            });
+            db()
+                .prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
+                .run(file.version.id, file.script.id);
+        }
+        const job = await queue();
+        await advance();
+        await acknowledge(job.deploymentId);
+        db()
+            .prepare(
+                "INSERT INTO deployments(id,action,started_at) VALUES('files-rebuild','promote',?)"
+            )
+            .run(new Date().toISOString());
+        db()
+            .prepare(
+                "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('files-rebuild',?,4,'pending')"
+            )
+            .run(job.name);
+        const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
+        const changes = [
+            { kind: 'plugin', relPath: 'replace.dll', content: Buffer.from('requested plugin') },
+            { kind: 'plugin', relPath: 'remove.dll', content: null },
+            { kind: 'script', relPath: 'new.gsc', content: Buffer.from('new script') },
+            { kind: 'config', relPath: 'custom.cfg', content: Buffer.from('set custom 1') }
+        ] as const;
+        for (const change of changes)
+            queueFileChange({
+                sourceKind: 'campaign',
+                sourceId: 'files-rebuild',
+                server: job.name,
+                inventorySource: 'campaign',
+                ...change
+            });
+        const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
+        const rebuild = requestRestart(job.name, { kind: 'campaign', id: 'files-rebuild' });
+        await advance();
+        const { getHostOperation, reportHostOperation } =
+            await import('../../src/lib/server/hosts/index.js');
+        const staged = db()
+            .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
+            .get(rebuild.id) as { stage_operation_id: string };
+        reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
+        await advance();
+        const operation = (await operationFor(rebuild.id))!;
+        expect(operation.kind).toBe('instance.rebuild');
+        expect(operation.payload).not.toHaveProperty('unhealthyRecovery');
+        const { recipeForDeployment } =
+            await import('../../src/lib/server/fleet/launch-recipes.js');
+        const recipe = recipeForDeployment(rebuild.id)!.recipe;
+        for (const change of changes) {
+            const actual = recipe.files.find(
+                (file) => file.kind === change.kind && file.path === change.relPath
+            );
+            if (change.content === null) expect(actual).toBeUndefined();
+            else expect(Buffer.from(actual!.content!, 'base64')).toEqual(change.content);
+        }
+        const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{
+            id: number;
+            path: string;
+        }>;
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: { before: files.map((file) => ({ ...file, contentBase64: null })) }
+        });
+        remote.state = 'unhealthy';
+        await advance();
+        expect((await operationFor(rebuild.id))!.kind).toBe('instance.restart');
+        await acknowledge(rebuild.id);
+        const recovery = (await operationFor(rebuild.id))!;
+        expect(recovery.kind).toBe('instance.rebuild');
+        expect(recovery.payload).not.toHaveProperty('stageId');
+        expect(recovery.payload.files).toEqual(operation.payload.files);
+        remote.state = 'running';
+        await acknowledge(rebuild.id);
+        expect((await detail(rebuild.id)).state).toBe('done');
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        expect(nativeInstance(job.name)?.generation).toBe(3);
+    });
+
     it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
         const job = await queue();
         await advance();
@@ -1761,6 +1851,19 @@ describe('native instance provisioning', () => {
         });
         const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
         const restart = requestRestart(job.name, { kind: 'campaign', id: 'campaign' });
+        expect(restart.action).toBe('rebuild');
+        // Upgrade queued work from older controllers, but wait for a capable host.
+        db().prepare("UPDATE server_deployments SET action='restart' WHERE id=?").run(restart.id);
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        heartbeat(hostId, { ...inventory(), capabilities: [] });
+        await advance();
+        expect(await detail(restart.id)).toMatchObject({
+            action: 'rebuild',
+            state: 'waiting',
+            stage: expect.stringContaining('Update the host agent')
+        });
+        expect(await operationFor(restart.id)).toBeNull();
+        heartbeat(hostId, inventory());
         remote.players = 2;
         await advance();
         const staged = db()
@@ -1811,9 +1914,18 @@ describe('native instance provisioning', () => {
         await advance();
         const activation = (await operationFor(restart.id))!;
         expect(activation).toMatchObject({
-            kind: 'instance.restart',
+            kind: 'instance.rebuild',
             generation: 1,
-            payload: { stageId: restart.id }
+            payload: {
+                stageId: restart.id,
+                nextGeneration: 2,
+                files: expect.arrayContaining([
+                    expect.objectContaining({
+                        path: 'runtime/plutonium/storage/t5/raw/scripts/sp/hud.gsc',
+                        sha256: createHash('sha256').update(after).digest('hex')
+                    })
+                ])
+            }
         });
         expect(
             db()
@@ -3819,6 +3931,12 @@ describe('cancelled native mutation bookkeeping', () => {
                     ])
                 ).rejects.toThrow();
                 const activation = (await operationFor(restart.id))!;
+                expect(activation.kind).toBe('instance.rebuild');
+                expect(activation.payload).toMatchObject({
+                    stageId: restart.id,
+                    nextGeneration: activation.generation! + 1
+                });
+                expect(activation.payload).not.toHaveProperty('unhealthyRecovery');
                 reportHostOperation(hostId, activation.id, {
                     state: 'completed',
                     result: {

tests/unit/native-safety.test.ts

@@ -36,7 +36,7 @@ beforeEach(async () => {
         storageAvailable: 100_000_000_000,
         preparedRuntimeImages: ['plutainer:test'],
         libraries: [{ id: 'published-game-version', state: 'ready' }],
-        capabilities: ['instance.safety-override']
+        capabilities: ['instance.safety-override', 'instance.rebuild-files']
     });
     const { db } = await import('../../src/lib/server/db/index.js');
     instanceId = randomUUID();
@@ -105,6 +105,22 @@ async function stageCampaign() {
             "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign-race',?,1,'pending')"
         )
         .run(server);
+    const { createLibraryImport, updateHostLibraryCaches } =
+        await import('../../src/lib/server/game-files/index.js');
+    const library = createLibraryImport({ hostId, game: 't5' });
+    db()
+        .prepare(
+            "UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?"
+        )
+        .run('a'.repeat(64), new Date().toISOString(), library.id);
+    updateHostLibraryCaches(hostId, [
+        { libraryVersionId: library.id, status: 'ready', manifestSha256: 'a'.repeat(64) }
+    ]);
+    db().prepare('UPDATE native_instances SET library_id=? WHERE server=?').run(library.id, server);
+    const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
+    reserveServerKey(server, 't5', 'saved-key');
+    const backend = await import('../../src/lib/server/backends/native.js');
+    vi.spyOn(backend, 'runtimePasswordForServer').mockReturnValue('saved-password');
     const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
     queueFileChange({
         sourceKind: 'campaign',
@@ -413,7 +429,7 @@ describe('provisioning admin override', () => {
         });
     });
 
-    it('overrides a waiting campaign restart while keeping its staged changes attached', async () => {
+    it('overrides a waiting campaign rebuild while keeping its staged changes attached', async () => {
         const id = await stageCampaign();
         const status = await import('../../src/lib/server/fleet/status-evidence.js');
         vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({
@@ -426,9 +442,11 @@ describe('provisioning admin override', () => {
         await confirm(id);
         await advance();
         const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
-        expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({
-            stageId: id,
-            safetyOverride: true
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        expect(getDeployment(id)?.deployment).toMatchObject({ state: 'running', error: null });
+        expect(getHostOperation((await operationId(id))!)).toMatchObject({
+            kind: 'instance.rebuild',
+            payload: { stageId: id, safetyOverride: true }
         });
         const { db } = await import('../../src/lib/server/db/index.js');
         expect(
@@ -518,7 +536,7 @@ describe('native scheduler ownership', () => {
         ).toHaveLength(2);
     });
 
-    it('holds a paused campaign and cancels withdrawn work before delivering a restart', async () => {
+    it('holds a paused campaign and cancels withdrawn work before delivering a rebuild', async () => {
         const id = await stageCampaign();
         const { db } = await import('../../src/lib/server/db/index.js');
         db()