Offer rebuild retry for failed creation commands

agent/ec_agent.py

@@ -10,6 +10,7 @@ import base64
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import contextlib
 from datetime import datetime, timezone
+import errno
 import math
 import hashlib
 import http.client
@@ -28,12 +29,13 @@ import subprocess
 import sys
 import threading
 import time
+import types
 import urllib.error
 import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.8.2"
+VERSION = "1.8.3"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -79,6 +81,106 @@ class AutoscalingInvalidated(Rejected):
     """This removal needs a new policy decision, never a later local retry."""
 
 
+OPERATION_ERROR_PHASES = {
+    "executing": "executing the operation",
+    "checking-instance": "checking the existing instance",
+    "validating-create-spec": "validating the container specification",
+    "preparing-create-directory": "preparing the instance directory",
+    "seeding-runtime": "seeding the runtime cache",
+    "writing-create-files": "writing initial files",
+    "persisting-create-metadata": "saving instance metadata",
+    "creating-container": "creating the container",
+    "starting-container": "starting the container",
+    "recording-created-instance": "recording the started container",
+    "inspecting-created-instance": "checking the created container",
+    "removing-old-instance": "removing the old instance directory",
+}
+
+
+def unexpected_operation_error(error, phase):
+    """Return bounded diagnostics without exception text, paths, or runtime data."""
+    def attribute(value, name):
+        try:
+            return getattr(value, name, None)
+        except Exception:
+            return None
+
+    def category(value):
+        # Names come from this fixed list, never an arbitrary exception class.
+        for kind in (urllib.error.HTTPError, urllib.error.URLError, http.client.IncompleteRead,
+                     http.client.RemoteDisconnected, http.client.HTTPException,
+                     subprocess.TimeoutExpired, TimeoutError,
+                     PermissionError, FileNotFoundError, FileExistsError, IsADirectoryError,
+                     NotADirectoryError, ConnectionError, OSError, sqlite3.OperationalError, sqlite3.Error,
+                     json.JSONDecodeError, UnicodeError, KeyError, TypeError, ValueError,
+                     MemoryError, RuntimeError):
+            if isinstance(value, kind):
+                return kind.__name__
+        return "Exception"
+
+    phase = phase if isinstance(phase, str) and phase in OPERATION_ERROR_PHASES else "executing"
+    details = {"type": category(error), "phase": phase, "agentVersion": VERSION}
+    evidence = error
+    if isinstance(error, urllib.error.HTTPError):
+        status = attribute(error, "code")
+        if type(status) is int and 100 <= status <= 599:
+            details["httpStatus"] = status
+    elif isinstance(error, urllib.error.URLError):
+        reason = attribute(error, "reason")
+        if isinstance(reason, BaseException):
+            details["causeType"] = category(reason)
+            evidence = reason
+    number = attribute(evidence, "errno") if isinstance(evidence, OSError) else None
+    number = number if type(number) is int and number in errno.errorcode else None
+    if number is not None:
+        details.update(errno=number, errnoName=errno.errorcode[number])
+
+    frames = []
+    trace = attribute(error, "__traceback__")
+    while isinstance(trace, types.TracebackType):
+        # Only agent-owned code locations are safe to report. Exclude source
+        # lines, filenames, other modules, frame locals, and exception chains.
+        if trace.tb_frame.f_globals is globals():
+            frames.append({"function": trace.tb_frame.f_code.co_name, "line": trace.tb_lineno})
+        trace = trace.tb_next
+    details["frames"] = frames[-8:]
+
+    if details.get("httpStatus") in (401, 403):
+        guidance = "Check host enrollment and controller artifact access permissions."
+    elif details.get("httpStatus") == 404:
+        guidance = "Check that the requested artifact is still available on the controller."
+    elif isinstance(error, urllib.error.HTTPError):
+        guidance = "Check controller availability and artifact delivery."
+    elif number == errno.ENOSPC:
+        guidance = "Check free disk space and inodes on the host."
+    elif number == errno.EDQUOT:
+        guidance = "Check the host filesystem quota."
+    elif number in (errno.EACCES, errno.EPERM) or isinstance(evidence, PermissionError):
+        guidance = "Check permissions on the host's managed files and directories."
+    elif number == errno.EROFS:
+        guidance = "Check whether the host filesystem is mounted read-only."
+    elif isinstance(error, (urllib.error.URLError, http.client.HTTPException)) or isinstance(evidence, (TimeoutError, ConnectionError)):
+        guidance = "Check host network, DNS, TLS, and controller connectivity."
+    elif isinstance(error, subprocess.TimeoutExpired):
+        guidance = "Check whether the host service command is stalled or unavailable."
+    elif isinstance(evidence, FileNotFoundError):
+        guidance = "Check that the required host files and directories are present."
+    elif isinstance(evidence, OSError):
+        guidance = "Check host storage, filesystem access, and service availability."
+    else:
+        guidance = "Inspect the saved operation diagnostics and host setup before retrying."
+    labels = [details["type"]]
+    if "httpStatus" in details:
+        labels.append(f"HTTP {details['httpStatus']}")
+    if "errnoName" in details:
+        labels.append(f"{details['errnoName']} ({details['errno']})")
+    if "causeType" in details:
+        labels.append("cause " + details["causeType"])
+    if frames:
+        labels.append(f"at {frames[-1]['function']}:{frames[-1]['line']}")
+    return f"Operation error while {OPERATION_ERROR_PHASES[phase]} ({'; '.join(labels)}). {guidance}", details
+
+
 def systemctl(*args):
     try:
         result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
@@ -962,6 +1064,7 @@ class Agent:
         self.plutonium_lock = threading.RLock()
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
         self.executing_id = None
+        self.operation_phase = None
         self.lane_lock = threading.Lock()
         self.lanes = {name: {"thread": None, "lastPollAt": None, "polled": time.monotonic(), "busy": False}
                       for name in ("health-1", "health-2", "read", "write")}
@@ -1767,6 +1870,7 @@ class Agent:
             return True
 
     def create(self, instance_id, generation, payload):
+        self.operation_phase = "checking-instance"
         self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
         current_marker = directory.parent / "current.json"
@@ -1779,12 +1883,17 @@ class Agent:
             if self.spec(instance_id, generation) != payload:
                 raise AgentError("Instance already exists with a different specification")
             return existing
+        self.operation_phase = "validating-create-spec"
         game, library, image, port, bind, env = self.validate_spec(payload)
         app = directory / "app"
+        self.operation_phase = "preparing-create-directory"
         app.mkdir(parents=True, exist_ok=True)
         self.make_writable(app, directory=True)
+        self.operation_phase = "seeding-runtime"
         self.seed_plutonium(app, image)
+        self.operation_phase = "writing-create-files"
         self.write_initial_files(app, payload.get("files", []))
+        self.operation_phase = "persisting-create-metadata"
         atomic_json(directory / "spec.json", payload)
         env_path = directory / "environment"
         atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode())
@@ -1800,12 +1909,16 @@ class Agent:
             args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
         args.append(image)
         self.check_creation_cancellation()
+        self.operation_phase = "creating-container"
         self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha})
         self.docker.call(*args, timeout=120)
         self.check_creation_cancellation()
+        self.operation_phase = "starting-container"
         self.docker.call("container", "start", name)
         self.check_creation_cancellation()
+        self.operation_phase = "recording-created-instance"
         atomic_json(current_marker, {"generation": generation})
+        self.operation_phase = "inspecting-created-instance"
         return self.inspect(instance_id, generation)
 
     def check_creation_cancellation(self):
@@ -1913,6 +2026,7 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 if kind == "instance.rebuild":
                     result = self.create(instance_id, next_generation, spec)
+                    self.operation_phase = "removing-old-instance"
                     old = self.instance_dir(instance_id, generation)
                     if old.exists():
                         shutil.rmtree(old, ignore_errors=False)
@@ -2507,6 +2621,7 @@ class Agent:
             except AgentError:
                 return self.journal.update(operation["id"], "accepted", {"blocked": "controller-unavailable", "message": "Waiting for controller contact before executing the operation"})
         self.executing_id = operation["id"]
+        self.operation_phase = "executing"
         try:
             result = self.dispatch(operation)
         except Held as err:
@@ -2518,11 +2633,17 @@ class Agent:
         except Exception as err:
             # The command may already have changed Docker or files. Never infer
             # failure means nothing happened, and never retry without inspection.
-            message = str(err) if isinstance(err, AgentError) else "Operation could not be completed; inspect its current state"
+            diagnostic = None
+            if isinstance(err, AgentError):
+                message = str(err)
+            else:
+                message, details = unexpected_operation_error(err, self.operation_phase)
+                diagnostic = {"unexpectedError": details}
             checkpoint = self.journal.get(operation["id"]).get("checkpoint")
-            return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
+            return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", diagnostic, error=message)
         finally:
             self.executing_id = None
+            self.operation_phase = None
         return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
 
     def recover_credential(self):

agent/tests/test_agent.py

@@ -404,6 +404,128 @@ class AgentTests(unittest.TestCase):
             with self.assertRaisesRegex(AgentError, "^Controller request failed$"):
                 self.agent.request("GET", "/api/agent/operations")
 
+    def test_unexpected_create_http_failure_reports_safe_diagnostics_and_does_not_replay(self):
+        secret = "private-token-must-not-appear"
+        error = module.urllib.error.HTTPError(f"https://user:{secret}@controller.example/blobs?token={secret}", 404,
+                                             secret, {"Authorization": "Bearer " + secret}, None)
+        error.__cause__ = RuntimeError(secret)
+        payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest",
+                   "port": 28960, "rconPassword": secret, "files": [{"path": "configs/server.cfg", "contentSha256": "a" * 64}]}
+        operation = self.operation("instance.create", payload)
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.side_effect = error
+            result = self.agent.execute(operation)
+            self.assertEqual(result["state"], "failed", result)
+            self.assertIsNone(result["checkpoint"])
+            self.assertEqual(self.agent.execute(operation), result)
+            opener.return_value.open.assert_called_once()
+        details = json.loads(result["result"])["unexpectedError"]
+        self.assertEqual(details["type"], "HTTPError")
+        self.assertEqual(details["httpStatus"], 404)
+        self.assertEqual(details["phase"], "validating-create-spec")
+        self.assertEqual(details["agentVersion"], module.VERSION)
+        self.assertEqual(details["frames"][-1]["function"], "artifact_open")
+        self.assertLessEqual(len(details["frames"]), 8)
+        self.assertTrue(all(set(frame) == {"function", "line"} and frame["line"] > 0 for frame in details["frames"]))
+        self.assertIn("validating the container specification", result["error"])
+        self.assertIn("HTTP 404", result["error"])
+        self.assertIn("at artifact_open:", result["error"])
+        self.assertIn("artifact is still available", result["error"])
+        self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls))
+        self.assertIsNone(self.agent.operation_phase)
+        self.agent.journal.close()
+        self.agent = Agent(self.config, self.docker)
+        self.assertEqual(self.agent.execute(operation), result)
+        with patch.object(self.agent, "request", return_value={}) as report:
+            self.agent.report(result)
+        receipt = report.call_args.args[2]
+        self.assertEqual(receipt["result"]["unexpectedError"], details)
+        safe_output = json.dumps(receipt)
+        for hidden in (secret, "controller.example", "Authorization", self.temp.name, "contentSha256", "rconPassword", "Traceback"):
+            self.assertNotIn(hidden, safe_output)
+
+    def test_unexpected_create_disk_failure_identifies_phase_before_mutation(self):
+        payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960}
+        with patch.object(self.agent, "seed_plutonium", side_effect=OSError(module.errno.ENOSPC, "private-error", "/private/secret-path")):
+            result = self.agent.execute(self.operation("instance.create", payload))
+        self.assertEqual(result["state"], "failed", result)
+        self.assertIsNone(result["checkpoint"])
+        details = json.loads(result["result"])["unexpectedError"]
+        self.assertEqual(details["phase"], "seeding-runtime")
+        self.assertEqual(details["errno"], module.errno.ENOSPC)
+        self.assertEqual(details["errnoName"], "ENOSPC")
+        self.assertIn("seeding the runtime cache", result["error"])
+        self.assertIn("free disk space and inodes", result["error"])
+        self.assertNotIn("private", result["error"] + result["result"])
+        self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls))
+
+    def test_unexpected_create_failure_after_checkpoint_remains_unknown_without_replay(self):
+        payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960}
+        docker_call = self.docker.call
+        creates = []
+
+        def failed_create(*args, **kwargs):
+            if args[:2] == ("container", "create"):
+                creates.append(args)
+                raise RuntimeError("private-runtime-data") from ValueError("private-cause")
+            return docker_call(*args, **kwargs)
+
+        operation = self.operation("instance.create", payload)
+        with patch.object(self.docker, "call", side_effect=failed_create):
+            result = self.agent.execute(operation)
+            self.assertEqual(result["state"], "unknown", result)
+            self.assertEqual(json.loads(result["checkpoint"])["phase"], "creating")
+            details = json.loads(result["result"])["unexpectedError"]
+            self.assertEqual(details["type"], "RuntimeError")
+            self.assertEqual(details["phase"], "creating-container")
+            self.assertEqual(self.agent.execute(operation), result)
+            self.assertEqual(len(creates), 1)
+        self.assertNotIn("private", result["error"] + result["result"])
+
+    def test_unexpected_error_guidance_uses_safe_exception_metadata(self):
+        for error, category, hint in (
+            (PermissionError(module.errno.EACCES, "private", "/private/path"), "PermissionError", "permissions"),
+            (OSError(module.errno.EDQUOT, "private"), "OSError", "quota"),
+            (OSError(module.errno.EROFS, "private"), "OSError", "read-only"),
+            (module.urllib.error.URLError(ConnectionRefusedError(module.errno.ECONNREFUSED, "private")), "URLError", "connectivity"),
+            (TimeoutError("private"), "TimeoutError", "connectivity"),
+            (http.client.IncompleteRead(b"private", 100), "IncompleteRead", "connectivity"),
+            (module.urllib.error.HTTPError("https://private", 403, "private", {}, None), "HTTPError", "enrollment"),
+            (module.urllib.error.HTTPError("https://private", 503, "private", {}, None), "HTTPError", "controller availability"),
+        ):
+            with self.subTest(category=category, hint=hint):
+                message, details = module.unexpected_operation_error(error, "executing")
+                self.assertEqual(details["type"], category)
+                self.assertIn(hint, message)
+                self.assertNotIn("private", message + json.dumps(details))
+
+    def test_unexpected_error_diagnostics_handle_unfamiliar_attributes_without_rendering_them(self):
+        class UnsafeValue:
+            def __str__(self):
+                raise AssertionError("private attribute was rendered")
+
+            def __eq__(self, other):
+                raise AssertionError("private attribute was compared")
+
+        class UnavailableTraceback(Exception):
+            def __getattribute__(self, name):
+                if name == "__traceback__":
+                    raise RuntimeError("private traceback access")
+                return super().__getattribute__(name)
+
+        filesystem_error = OSError("private error")
+        filesystem_error.errno = UnsafeValue()
+        errors = [filesystem_error, UnavailableTraceback("private exception"), type("private class", (Exception,), {})("private args"),
+                  module.urllib.error.URLError(UnsafeValue()),
+                  module.urllib.error.HTTPError("https://private", UnsafeValue(), "private reason", {}, None)]
+        for error in errors:
+            error.__cause__ = RuntimeError("private cause")
+            message, details = module.unexpected_operation_error(error, "private phase")
+            self.assertEqual(details["phase"], "executing")
+            self.assertNotIn("httpStatus", details)
+            self.assertNotIn("errno", details)
+            self.assertNotIn("private", message + json.dumps(details))
+
     def test_read_and_health_lanes_retry_unexpected_failures_without_logging_secrets(self):
         for lane in ("read", "health"):
             with self.subTest(lane=lane):

docs/guides/collections.md

@@ -287,12 +287,22 @@ server launch until the health checks pass.
 
 For failed or interrupted creations, **Resume health verification** appears only
 when the original server is still registered on its assigned port and no later
-lifecycle operation has targeted that ID. It resumes observation only; it never
-recreates or restarts a container. Missing servers and later destruction attempts
-require inspection rather than replaying provisioning automatically.
-After an application restart, the worker automatically resumes observation of
-eligible interrupted startups so their occupied slots are retained. It never
-repeats container creation or deployment as part of this recovery.
+lifecycle operation has targeted that ID. Native commands must have completed
+before readiness verification can resume. If the container is unhealthy, the
+operation may continue its remaining automatic restart and rebuild recovery steps.
+
+When the host conclusively reports that the initial creation command failed,
+**Retry with rebuild** starts a new provisioning operation for the same lobby and
+port. It deletes the container and instance files and uses the latest promoted
+setup. Normal player safety, capacity, and protected work checks still apply.
+The original failure stays in history and links to the recovery operation.
+This retry is unavailable for unknown command results, replacement instances,
+cancelled creation cleanup, or a later operation on the same server.
+
+After an application restart, the worker reconciles the original host command
+before continuing eligible interrupted startups. Unknown command results retain
+their original command ID and occupied slots until the host confirms the outcome.
+Missing server registrations and later destruction attempts require inspection.
 
 ## API
 
@@ -339,7 +349,9 @@ there while preserving query parameters.
 - `POST /api/servers`: existing request plus optional `previewId`; the game and
   map must match that preview. `expectedName` remains supported.
 - `GET /api/deployments/:id`: includes `recipe`, `blueprintName`, and
-  `canResumeVerification` for lifecycle operations.
+  `canResumeVerification` and `canRetryWithRebuild` for lifecycle operations.
 - `POST /api/deployments/:id/resume`: resume eligible creation health checks.
+- `POST /api/provisioning/:id/retry-rebuild`: request a new rebuild for an eligible
+  failed creation and link it from the original operation.
 
 Collection launches begin processing immediately after all requests are queued.

src/lib/provisioning-detail.ts

@@ -101,6 +101,7 @@ export interface ProvisioningDetail {
     safetyOverride?: { actorId: string; actorName: string; confirmedAt: string } | null;
     canCancel?: boolean;
     canCancelCreation?: boolean;
+    canRetryWithRebuild?: boolean;
     cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null;
     backend?: 'native';
     workerActivity?: WorkerActivity[];

src/lib/server/provisioning/history.ts

@@ -14,6 +14,7 @@ import { canCancelCreation } from './cancel-creation.js';
 import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js';
 import { committedRebalanceStartup } from './concurrency.js';
 import { nativeDeployment } from '../backends/native.js';
+import { ensureHostSchema } from '../hosts/schema.js';
 import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
 import { serverTitle } from '../../ui/server-labels.js';
 import { provisioningQueueStatus } from '../../ui/provisioning-queue.js';
@@ -149,7 +150,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
             recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null,
             canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
-            canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
+            canResumeRebuild: canResumeRebuild(id), canRetryWithRebuild: canRetryWithRebuild(id),
+            canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
             progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
             warningMessages: warningMessages(deployment.warnings), generatedAt };
     })();
@@ -197,13 +199,39 @@ export function resumeCleanup(id: string): boolean {
 }
 
 export function canResumeVerification(id: string): boolean {
+    ensureHostSchema(db());
     return !!db().prepare(`SELECT d.id FROM server_deployments d
         JOIN server_creation_jobs j ON j.deployment_id = d.id
-        JOIN servers s ON s.name = j.server AND s.game_port = j.port
-        WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND (EXISTS (
-            SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL) OR EXISTS (
-            SELECT 1 FROM ec_host_migrations m JOIN native_provisioning n ON n.deployment_id=m.deployment_id
-                WHERE m.deployment_id=d.id AND m.removed_at IS NOT NULL AND n.operation_id IS NOT NULL)))) AND d.state IN ('failed', 'interrupted')
+        JOIN servers s ON s.name = j.server AND s.game_port = j.port AND s.destroyed_at IS NULL
+        LEFT JOIN native_provisioning n ON n.deployment_id=d.id
+        LEFT JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
+        LEFT JOIN ec_host_operations o ON o.id=n.operation_id
+        WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('create','rebuild')
+        AND d.state IN ('failed', 'interrupted')
+        AND ((n.deployment_id IS NOT NULL AND i.port=j.port AND o.state='completed'
+            AND o.instance_id=i.id AND o.host_id=i.host_id AND o.generation=n.generation
+            AND ((o.kind IN ('instance.create','instance.restart') AND i.generation=n.generation)
+                OR (o.kind='instance.rebuild' AND i.generation IN (n.generation,n.generation+1))))
+            OR (n.deployment_id IS NULL AND (d.action='create' OR EXISTS (
+                SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id=d.id AND r.removed_at IS NOT NULL))))
+        AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
+        .get(id);
+}
+
+/** A conclusively failed initial create can enter the normal rebuild flow. */
+export function canRetryWithRebuild(id: string): boolean {
+    ensureHostSchema(db());
+    return !!db().prepare(`SELECT d.id FROM server_deployments d
+        JOIN server_creation_jobs j ON j.deployment_id=d.id
+        JOIN servers s ON s.name=d.server AND s.game_port=j.port AND s.destroyed_at IS NULL
+        JOIN native_provisioning n ON n.deployment_id=d.id AND n.recovery_step IS NULL
+        JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
+            AND i.generation=n.generation AND i.port=j.port
+        JOIN ec_host_operations o ON o.id=n.operation_id AND o.instance_id=i.id
+            AND o.host_id=i.host_id AND o.generation=n.generation
+        WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action='create'
+        AND d.state IN ('failed','interrupted') AND o.kind='instance.create' AND o.state='failed'
+        AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id)
         AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)`)
         .get(id);
 }
@@ -226,7 +254,7 @@ export function resumeRebuild(id: string): boolean {
     })();
 }
 
-/** Resume observation only. Never repeat a container creation or restart. */
+/** Resume readiness after a completed command; unhealthy recovery still applies. */
 export function resumeVerification(id: string): boolean {
     return db().transaction(() => {
         if (!canResumeVerification(id)) return false;

src/lib/server/provisioning/index.ts

@@ -1,6 +1,6 @@
 // Request/observation boundary used by APIs, campaigns, beta tests, and schedules.
 // Execution is driven exclusively by the provisioning worker.
-export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
+export { requestLifecycle, requestRetryWithRebuild, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
 export type { LifecycleOperation, RestartSource } from './requests.js';
 export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextLobbyNumber } from './instances.js';
 export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';

src/lib/server/provisioning/requests.ts

@@ -3,7 +3,7 @@ import { currentRequestUsername } from '../auth/request-context.js';
 import { maintenanceAllowed } from '../maintenance/policies.js';
 import { db } from '../db/index.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
-import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
+import { beginDeployment, canRetryWithRebuild, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
 import { queueRebuildServer } from './instances.js';
 import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
 import { recordProgress } from './progress.js';
@@ -166,6 +166,23 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
 
 type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string };
 
+/** Keep the failed launch immutable and let normal rebuild admission own recovery. */
+export function requestRetryWithRebuild(id: string): LifecycleRequestResult {
+    return db().transaction(() => {
+        if (!canRetryWithRebuild(id)) return { ok: false as const, status: 409,
+            error: 'This failed creation can no longer be retried with a rebuild. Refresh and inspect its current operation.' };
+        const original = provisioningOperation(id)!;
+        const result = requestLifecycle(original.server, 'rebuild');
+        if (!result.ok) return result;
+        recordProgress(id, { recovery: { state: 'transferred', operationId: result.deploymentId,
+            message: 'A new rebuild was requested for this server. The failed creation and its error are preserved.' },
+            event: { kind: 'retry-requested', message: 'Retry with rebuild requested', details: { operationId: result.deploymentId } } });
+        recordProgress(result.deploymentId, { event: { kind: 'retry-requested',
+            message: 'Rebuild requested after failed creation', details: { previousOperationId: id } } });
+        return result;
+    })();
+}
+
 function watchdogRequestGuard(server: string, incidentId: string): Extract<LifecycleRequestResult, { ok: false }> | null {
     const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined;
     if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' };

src/lib/ui/provisioning-presentation.ts

@@ -76,11 +76,13 @@ export function provisioningOutcome(detail: ProvisioningDetail) {
     let description: string;
     if (detail.canResumeCleanup) description = 'The remote deployment was deleted. Resume the remaining local cleanup without repeating container removal.';
     else if (detail.canResumeRebuild) description = 'The remote deployment was deleted. Resume local cleanup, then create a fresh instance in the reserved server slot.';
-    else if (detail.canResumeVerification) description = 'Resume observation of the existing instance. This does not create, restart, or rebuild a container.';
+    else if (detail.canResumeVerification) description = 'Resume readiness checks for the existing instance. An unhealthy container may continue through this operation’s remaining restart and rebuild recovery steps.';
+    else if (detail.canRetryWithRebuild) description = 'The host reported that creation failed. Retry with rebuild starts a new attempt for the same lobby and port using the latest promoted setup.';
     else if (detail.cancellation) description = detail.cancellation.state === 'done'
         ? 'Creation was cancelled. Its instance files, container, and port reservations have been removed.'
         : 'Creation was cancelled. Cleanup is tracked separately below; ports remain reserved until the host confirms removal.';
     else if (operation.state === 'cancelled') description = 'This request is cancelled. Recorded prior work is preserved below; remaining work will not run.';
+    else if (failure && progress?.recovery?.state === 'transferred' && progress.recovery.operationId && progress.recovery.operationId !== operation.id) description = 'A recovery operation has been requested for this server. Follow the linked operation below; this record keeps the original failure.';
     else if (failure) description = progress?.command?.state === 'uncertain'
         ? 'The remote response was uncertain. Inspect the recorded result and server before requesting another operation.'
         : 'Review the recorded failure and execution evidence before submitting another operation.';
@@ -162,6 +164,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi
     if (!record(value) || !record(value.deployment)) return false;
     if (value.canCancel !== undefined && typeof value.canCancel !== 'boolean') return false;
     if (value.canCancelCreation !== undefined && typeof value.canCancelCreation !== 'boolean') return false;
+    if (value.canRetryWithRebuild !== undefined && typeof value.canRetryWithRebuild !== 'boolean') return false;
     if (value.cancellation !== undefined && value.cancellation !== null && !(record(value.cancellation)
         && text(value.cancellation.cleanupId) && text(value.cancellation.state) && text(value.cancellation.stage)
         && nullableText(value.cancellation.error))) return false;

src/lib/ui/provisioning-refresh.ts

@@ -1,4 +1,4 @@
-export type ProvisioningMutation = 'resume' | 'cancel' | 'cancel-creation' | 'override';
+export type ProvisioningMutation = 'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override';
 
 export interface ProvisioningRefreshState<T> {
     id: string;

src/routes/api/provisioning/[id]/retry-rebuild/+server.ts

@@ -0,0 +1,10 @@
+import { json } from '@sveltejs/kit';
+import { requestRetryWithRebuild } from '$lib/server/provisioning/index.js';
+import { kickProvisioning } from '$lib/server/pipeline/worker.js';
+
+export function POST({ params }: { params: { id: string } }) {
+    const result = requestRetryWithRebuild(params.id);
+    if (!result.ok) return json({ error: result.error }, { status: result.status });
+    void kickProvisioning().catch(err => console.error('[provisioning] rebuild retry kick failed:', err));
+    return json(result, { status: 202 });
+}

src/routes/provisioning/[id]/+page.svelte

@@ -59,6 +59,17 @@
     });
     onMount(() => pollVisible(() => { now = Date.now(); return controller.refresh(); }, 5000, false));
     onDestroy(() => controller.dispose());
+    async function retryWithRebuild() {
+        if (!deployment || !detail?.canRetryWithRebuild || view.busy) return;
+        const id = deployment.id;
+        const confirmed = await confirmAction({
+            title: 'Retry creation with a rebuild?',
+            message: `Rebuild ${serverTitle(deployment.server)} (${deployment.server}) using the latest promoted setup. This keeps the same lobby and port, deletes its container and all instance files, and starts a new provisioning operation.\n\nPlayer safety, capacity limits, and protected work checks still apply. The original failure remains in this operation’s history.`,
+            confirmLabel: 'Retry with rebuild', danger: true
+        });
+        if (confirmed && view.id === id && detail?.canRetryWithRebuild) await controller.act('retry-rebuild',
+            'Rebuild retry requested. Follow the linked recovery operation below for progress.');
+    }
     async function overrideSafety() {
         if (!deployment || !canOverride || view.busy || detail?.safetyOverrideUnavailableReason) return;
         const id = deployment.id;
@@ -107,10 +118,11 @@
             {#if detail.warningMessages.length}<div class="warnings"><h3>Recorded warnings</h3><ul>{#each detail.warningMessages as warning}<li>{warning}</li>{/each}</ul></div>{/if}
 
             {#if detail.safetyOverride}<div class="warnings"><h3>Admin override enabled</h3><p>Player safety checks bypassed for this operation. Confirmed by {detail.safetyOverride.actorName} · <RelativeTime value={detail.safetyOverride.confirmedAt} />.</p></div>{/if}
-            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
+            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canRetryWithRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
                 <div class="recovery-actions">
                     {#if canOverride}<div><Button variant="danger" disabled={!!view.busy || !!detail.safetyOverrideUnavailableReason} loading={view.busy === 'override'} onclick={overrideSafety}>Admin override…</Button><p class="action-help">{detail.safetyOverrideUnavailableReason ?? 'Confirm to bypass player safety checks for this operation.'}</p></div>{/if}
-                    {#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. The existing instance will be observed without another creation or restart.')}>Resume health verification</Button>{/if}
+                    {#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. An unhealthy container may continue through this operation’s remaining recovery steps.')}>Resume health verification</Button>{/if}
+                    {#if detail.canRetryWithRebuild}<div><Button variant="primary" loading={view.busy === 'retry-rebuild'} disabled={!!view.busy} onclick={retryWithRebuild}>Retry with rebuild</Button><p class="action-help">Starts a new rebuild for the same lobby and port. Deletes instance files and uses the latest promoted setup.</p></div>{/if}
                     {#if detail.canResumeRebuild}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Rebuild cleanup requested. Fresh creation waits for local cleanup to finish.')}>Resume rebuild cleanup</Button>{/if}
                     {#if detail.canResumeCleanup}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Local cleanup requested. Remote container deletion will not be repeated.')}>Resume cleanup</Button>{/if}
                     {#if detail.canCancel}<div><Button disabled={!!view.busy} loading={view.busy === 'cancel'} onclick={() => controller.act('cancel', 'Operation cancelled. Already-submitted commands may finish; no further steps will be started.')}>Cancel operation</Button><p class="action-help">Stops remaining work and health checks. Changes already made are not undone.</p></div>{/if}

static/agent/ec-agent.py

@@ -10,6 +10,7 @@ import base64
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import contextlib
 from datetime import datetime, timezone
+import errno
 import math
 import hashlib
 import http.client
@@ -28,12 +29,13 @@ import subprocess
 import sys
 import threading
 import time
+import types
 import urllib.error
 import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.8.2"
+VERSION = "1.8.3"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -79,6 +81,106 @@ class AutoscalingInvalidated(Rejected):
     """This removal needs a new policy decision, never a later local retry."""
 
 
+OPERATION_ERROR_PHASES = {
+    "executing": "executing the operation",
+    "checking-instance": "checking the existing instance",
+    "validating-create-spec": "validating the container specification",
+    "preparing-create-directory": "preparing the instance directory",
+    "seeding-runtime": "seeding the runtime cache",
+    "writing-create-files": "writing initial files",
+    "persisting-create-metadata": "saving instance metadata",
+    "creating-container": "creating the container",
+    "starting-container": "starting the container",
+    "recording-created-instance": "recording the started container",
+    "inspecting-created-instance": "checking the created container",
+    "removing-old-instance": "removing the old instance directory",
+}
+
+
+def unexpected_operation_error(error, phase):
+    """Return bounded diagnostics without exception text, paths, or runtime data."""
+    def attribute(value, name):
+        try:
+            return getattr(value, name, None)
+        except Exception:
+            return None
+
+    def category(value):
+        # Names come from this fixed list, never an arbitrary exception class.
+        for kind in (urllib.error.HTTPError, urllib.error.URLError, http.client.IncompleteRead,
+                     http.client.RemoteDisconnected, http.client.HTTPException,
+                     subprocess.TimeoutExpired, TimeoutError,
+                     PermissionError, FileNotFoundError, FileExistsError, IsADirectoryError,
+                     NotADirectoryError, ConnectionError, OSError, sqlite3.OperationalError, sqlite3.Error,
+                     json.JSONDecodeError, UnicodeError, KeyError, TypeError, ValueError,
+                     MemoryError, RuntimeError):
+            if isinstance(value, kind):
+                return kind.__name__
+        return "Exception"
+
+    phase = phase if isinstance(phase, str) and phase in OPERATION_ERROR_PHASES else "executing"
+    details = {"type": category(error), "phase": phase, "agentVersion": VERSION}
+    evidence = error
+    if isinstance(error, urllib.error.HTTPError):
+        status = attribute(error, "code")
+        if type(status) is int and 100 <= status <= 599:
+            details["httpStatus"] = status
+    elif isinstance(error, urllib.error.URLError):
+        reason = attribute(error, "reason")
+        if isinstance(reason, BaseException):
+            details["causeType"] = category(reason)
+            evidence = reason
+    number = attribute(evidence, "errno") if isinstance(evidence, OSError) else None
+    number = number if type(number) is int and number in errno.errorcode else None
+    if number is not None:
+        details.update(errno=number, errnoName=errno.errorcode[number])
+
+    frames = []
+    trace = attribute(error, "__traceback__")
+    while isinstance(trace, types.TracebackType):
+        # Only agent-owned code locations are safe to report. Exclude source
+        # lines, filenames, other modules, frame locals, and exception chains.
+        if trace.tb_frame.f_globals is globals():
+            frames.append({"function": trace.tb_frame.f_code.co_name, "line": trace.tb_lineno})
+        trace = trace.tb_next
+    details["frames"] = frames[-8:]
+
+    if details.get("httpStatus") in (401, 403):
+        guidance = "Check host enrollment and controller artifact access permissions."
+    elif details.get("httpStatus") == 404:
+        guidance = "Check that the requested artifact is still available on the controller."
+    elif isinstance(error, urllib.error.HTTPError):
+        guidance = "Check controller availability and artifact delivery."
+    elif number == errno.ENOSPC:
+        guidance = "Check free disk space and inodes on the host."
+    elif number == errno.EDQUOT:
+        guidance = "Check the host filesystem quota."
+    elif number in (errno.EACCES, errno.EPERM) or isinstance(evidence, PermissionError):
+        guidance = "Check permissions on the host's managed files and directories."
+    elif number == errno.EROFS:
+        guidance = "Check whether the host filesystem is mounted read-only."
+    elif isinstance(error, (urllib.error.URLError, http.client.HTTPException)) or isinstance(evidence, (TimeoutError, ConnectionError)):
+        guidance = "Check host network, DNS, TLS, and controller connectivity."
+    elif isinstance(error, subprocess.TimeoutExpired):
+        guidance = "Check whether the host service command is stalled or unavailable."
+    elif isinstance(evidence, FileNotFoundError):
+        guidance = "Check that the required host files and directories are present."
+    elif isinstance(evidence, OSError):
+        guidance = "Check host storage, filesystem access, and service availability."
+    else:
+        guidance = "Inspect the saved operation diagnostics and host setup before retrying."
+    labels = [details["type"]]
+    if "httpStatus" in details:
+        labels.append(f"HTTP {details['httpStatus']}")
+    if "errnoName" in details:
+        labels.append(f"{details['errnoName']} ({details['errno']})")
+    if "causeType" in details:
+        labels.append("cause " + details["causeType"])
+    if frames:
+        labels.append(f"at {frames[-1]['function']}:{frames[-1]['line']}")
+    return f"Operation error while {OPERATION_ERROR_PHASES[phase]} ({'; '.join(labels)}). {guidance}", details
+
+
 def systemctl(*args):
     try:
         result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
@@ -962,6 +1064,7 @@ class Agent:
         self.plutonium_lock = threading.RLock()
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
         self.executing_id = None
+        self.operation_phase = None
         self.lane_lock = threading.Lock()
         self.lanes = {name: {"thread": None, "lastPollAt": None, "polled": time.monotonic(), "busy": False}
                       for name in ("health-1", "health-2", "read", "write")}
@@ -1767,6 +1870,7 @@ class Agent:
             return True
 
     def create(self, instance_id, generation, payload):
+        self.operation_phase = "checking-instance"
         self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
         current_marker = directory.parent / "current.json"
@@ -1779,12 +1883,17 @@ class Agent:
             if self.spec(instance_id, generation) != payload:
                 raise AgentError("Instance already exists with a different specification")
             return existing
+        self.operation_phase = "validating-create-spec"
         game, library, image, port, bind, env = self.validate_spec(payload)
         app = directory / "app"
+        self.operation_phase = "preparing-create-directory"
         app.mkdir(parents=True, exist_ok=True)
         self.make_writable(app, directory=True)
+        self.operation_phase = "seeding-runtime"
         self.seed_plutonium(app, image)
+        self.operation_phase = "writing-create-files"
         self.write_initial_files(app, payload.get("files", []))
+        self.operation_phase = "persisting-create-metadata"
         atomic_json(directory / "spec.json", payload)
         env_path = directory / "environment"
         atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode())
@@ -1800,12 +1909,16 @@ class Agent:
             args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
         args.append(image)
         self.check_creation_cancellation()
+        self.operation_phase = "creating-container"
         self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha})
         self.docker.call(*args, timeout=120)
         self.check_creation_cancellation()
+        self.operation_phase = "starting-container"
         self.docker.call("container", "start", name)
         self.check_creation_cancellation()
+        self.operation_phase = "recording-created-instance"
         atomic_json(current_marker, {"generation": generation})
+        self.operation_phase = "inspecting-created-instance"
         return self.inspect(instance_id, generation)
 
     def check_creation_cancellation(self):
@@ -1913,6 +2026,7 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 if kind == "instance.rebuild":
                     result = self.create(instance_id, next_generation, spec)
+                    self.operation_phase = "removing-old-instance"
                     old = self.instance_dir(instance_id, generation)
                     if old.exists():
                         shutil.rmtree(old, ignore_errors=False)
@@ -2507,6 +2621,7 @@ class Agent:
             except AgentError:
                 return self.journal.update(operation["id"], "accepted", {"blocked": "controller-unavailable", "message": "Waiting for controller contact before executing the operation"})
         self.executing_id = operation["id"]
+        self.operation_phase = "executing"
         try:
             result = self.dispatch(operation)
         except Held as err:
@@ -2518,11 +2633,17 @@ class Agent:
         except Exception as err:
             # The command may already have changed Docker or files. Never infer
             # failure means nothing happened, and never retry without inspection.
-            message = str(err) if isinstance(err, AgentError) else "Operation could not be completed; inspect its current state"
+            diagnostic = None
+            if isinstance(err, AgentError):
+                message = str(err)
+            else:
+                message, details = unexpected_operation_error(err, self.operation_phase)
+                diagnostic = {"unexpectedError": details}
             checkpoint = self.journal.get(operation["id"]).get("checkpoint")
-            return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
+            return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", diagnostic, error=message)
         finally:
             self.executing_id = None
+            self.operation_phase = None
         return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
 
     def recover_credential(self):

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.8.2",
+  "version": "1.8.3",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "9715027ee71b4bab7df3ec44186eb81554f913741f1dde3e5d79b7e525a32ba6",
+  "sha256": "388569ffd4566dee63584970838d4d3ae7d2b8ef92675ae7627c31389dcbfec0",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/native-provisioning.test.ts

@@ -1503,6 +1503,169 @@ describe('automatic capacity admission and placement', () => {
     });
 });
 
+describe('native verification resume and explicit failed-creation retry', () => {
+    async function failedCreation(state: 'failed' | 'interrupted' = 'failed') {
+        const job = await queue(); await advance();
+        const original = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, original.id, { state: 'failed', error: 'Host runtime preflight failed' });
+        await advance();
+        if (state === 'interrupted') {
+            const { db } = await import('../../src/lib/server/db/index.js');
+            db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId);
+        }
+        return { job, original };
+    }
+
+    it.each(['failed', 'interrupted'] as const)('retries a %s creation through a new rebuild while retaining the original failure', async state => {
+        const { job, original } = await failedCreation(state);
+        const history = await import('../../src/lib/server/provisioning/history.js');
+        expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: true });
+        const { POST: resume } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
+        expect(resume({ params: { id: job.deploymentId } }).status).toBe(409);
+        const { POST: retry } = await import('../../src/routes/api/provisioning/[id]/retry-rebuild/+server.js');
+        const response = await retry({ params: { id: job.deploymentId } });
+        expect(response.status).toBe(202);
+        const next = await response.json() as { ok: boolean; deploymentId: string };
+        expect(next.ok).toBe(true);
+        expect(next.deploymentId).not.toBe(job.deploymentId);
+        expect(await detail(next.deploymentId)).toMatchObject({ action: 'rebuild', state: 'waiting', server: job.name });
+        expect(history.getDeployment(job.deploymentId)).toMatchObject({
+            canRetryWithRebuild: false,
+            deployment: { state, error: 'Host runtime preflight failed' },
+            progress: { recovery: { state: 'transferred', operationId: next.deploymentId } }
+        });
+        expect((await retry({ params: { id: job.deploymentId } })).status).toBe(409);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server=?').get(job.name)).toEqual({ count: 2 });
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        expect(nativeInstance(job.name)).toMatchObject({ id: original.instanceId, generation: 1, port: original.payload.port });
+        remote.state = 'missing'; await advance();
+        const rebuilding = (await operationFor(next.deploymentId))!;
+        expect(rebuilding).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: original.instanceId,
+            payload: { name: job.name, port: original.payload.port, nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
+        expect(rebuilding.id).not.toBe(original.id);
+        expect(nativeInstance(job.name)?.generation).toBe(1);
+        remote.state = 'running'; await acknowledge(next.deploymentId);
+        expect((await detail(next.deploymentId)).state).toBe('done');
+        expect(nativeInstance(job.name)?.generation).toBe(2);
+        // An interrupted original is reconciled to its already-failed receipt
+        // on the next worker pass; the failure itself is never cleared.
+        expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: 'Host runtime preflight failed' });
+        expect(await operationFor(job.deploymentId)).toMatchObject({ id: original.id, state: 'failed' });
+        expect(db().prepare("SELECT kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.rebuild','instance.restart') ORDER BY rowid").all())
+            .toEqual([{ kind: 'instance.create' }, { kind: 'instance.rebuild' }]);
+    });
+
+    it.each([
+        ['create', false], ['create', true], ['rebuild', false], ['rebuild', true],
+        ['recovery-restart', false], ['recovery-restart', true], ['recovery-rebuild', false], ['recovery-rebuild', true]
+    ] as const)('resumes completed %s verification with the same command (receipt applied=%s)', async (action, applied) => {
+        let job = await queue(); await advance();
+        if (action === 'rebuild') {
+            await acknowledge(job.deploymentId);
+            const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+            const rebuild = queueNativeRebuild(job.name, 'test');
+            if (!rebuild.ok) throw new Error(rebuild.message);
+            job = rebuild; await advance();
+        } else if (action.startsWith('recovery')) {
+            remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
+            if (action === 'recovery-rebuild') await acknowledge(job.deploymentId);
+        }
+        const operation = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        remote.state = 'starting'; reportHostOperation(hostId, operation.id, { state: 'completed' });
+        if (applied) await advance();
+        const history = await import('../../src/lib/server/provisioning/history.js');
+        history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted for inspection' });
+        expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: true, canRetryWithRebuild: false });
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const commands = db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all();
+        const { POST } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
+        expect(POST({ params: { id: job.deploymentId } }).status).toBe(202);
+        remote.state = 'running'; await advance();
+        expect((await detail(job.deploymentId)).state).toBe('done');
+        expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
+        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all()).toEqual(commands);
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        expect(nativeInstance(job.name)?.generation).toBe(operation.generation! + (operation.kind === 'instance.rebuild' ? 1 : 0));
+        expect(history.getDeployment(job.deploymentId)!.events).toContainEqual(expect.objectContaining({
+            kind: 'outcome', message: 'Failed: Verification interrupted for inspection'
+        }));
+    });
+
+    it.each(['pending', 'accepted', 'running', 'unknown', 'failed'] as const)('does not offer verification of a %s command or replay an ambiguous result', async state => {
+        const job = await queue(); await advance();
+        const operation = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        if (state !== 'pending') reportHostOperation(hostId, operation.id, { state });
+        const history = await import('../../src/lib/server/provisioning/history.js');
+        history.finishDeployment(job.deploymentId, { ok: false, message: 'Controller observation interrupted' });
+        expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: state === 'failed' });
+        expect(history.resumeVerification(job.deploymentId)).toBe(false);
+        if (state !== 'failed') {
+            const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js');
+            expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 });
+        }
+        const { db } = await import('../../src/lib/server/db/index.js');
+        if (state === 'unknown') db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId);
+        await advance();
+        expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
+        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").all())
+            .toEqual([{ id: operation.id }]);
+    });
+
+    it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work', 'creation cleanup', 'recovery attempt'])
+        ('refuses to retry a failed creation after its %s changes', async change => {
+            const { job, original } = await failedCreation();
+            const { db } = await import('../../src/lib/server/db/index.js');
+            if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
+            if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id);
+            if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id);
+            if (change === 'command host') {
+                const { createHost } = await import('../../src/lib/server/hosts/index.js');
+                db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id);
+            }
+            if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name);
+            if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
+            if (change === 'newer work') {
+                const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
+                beginDeployment(job.name, 'restart');
+            }
+            if (change === 'creation cleanup') {
+                const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
+                expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
+            }
+            if (change === 'recovery attempt') db().prepare("UPDATE native_provisioning SET recovery_step='restart' WHERE deployment_id=?").run(job.deploymentId);
+            const history = await import('../../src/lib/server/provisioning/history.js');
+            const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js');
+            expect(history.getDeployment(job.deploymentId)?.canRetryWithRebuild).toBe(false);
+            expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 });
+            expect(db().prepare("SELECT id FROM server_deployments WHERE action='rebuild'").all()).toEqual([]);
+        });
+
+    it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work'])
+        ('refuses to resume a completed creation after its %s changes', async change => {
+            const job = await queue(); await advance();
+            const original = (await operationFor(job.deploymentId))!;
+            const { reportHostOperation, createHost } = await import('../../src/lib/server/hosts/index.js');
+            reportHostOperation(hostId, original.id, { state: 'completed' });
+            const history = await import('../../src/lib/server/provisioning/history.js');
+            history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted' });
+            const { db } = await import('../../src/lib/server/db/index.js');
+            if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
+            if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id);
+            if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id);
+            if (change === 'command host') db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id);
+            if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name);
+            if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
+            if (change === 'newer work') history.beginDeployment(job.name, 'restart');
+            expect(history.getDeployment(job.deploymentId)?.canResumeVerification).toBe(false);
+            expect(history.resumeVerification(job.deploymentId)).toBe(false);
+            expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
+        });
+});
+
 describe('unhealthy recovery within native create and rebuild operations', () => {
     async function launch(action: 'create' | 'rebuild') {
         const job = await queue(); await advance();

tests/unit/provisioning-presentation.test.ts

@@ -19,6 +19,20 @@ function fixture(): ProvisioningDetail {
 }
 
 describe('provisioning outcome and evidence presentation', () => {
+    it('distinguishes a failed creation retry from readiness verification and its linked recovery', () => {
+        const detail = fixture();
+        detail.deployment.state = 'failed';
+        detail.canRetryWithRebuild = true;
+        expect(provisioningOutcome(detail)).toMatchObject({ active: false, headline: 'Create failed' });
+        expect(provisioningOutcome(detail).description).toContain('new attempt for the same lobby and port');
+        detail.canRetryWithRebuild = false;
+        detail.progress = progress({ recovery: { state: 'transferred', operationId: 'recovery-a', at, message: 'Rebuild requested' } });
+        expect(provisioningOutcome(detail).headline).toBe('Create failed');
+        expect(provisioningOutcome(detail).description).toContain('linked operation');
+        detail.progress = progress();
+        detail.canResumeVerification = true;
+        expect(provisioningOutcome(detail).description).toContain('remaining restart and rebuild recovery steps');
+    });
     it('distinguishes cancellation from confirmed cleanup and validates its link data', () => {
         const detail = fixture();
         detail.deployment.state = 'cancelled';
@@ -142,6 +156,15 @@ describe('provisioning outcome and evidence presentation', () => {
 });
 
 describe('provisioning response and return-context boundaries', () => {
+    it('accepts optional retry eligibility only as an authoritative boolean', () => {
+        const detail = fixture();
+        for (const canRetryWithRebuild of [undefined, false, true]) {
+            expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(true);
+        }
+        for (const canRetryWithRebuild of ['true', 1, null]) {
+            expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(false);
+        }
+    });
     it('accepts persisted usernames and legacy details but rejects malformed requesters', () => {
         const detail = fixture();
         for (const requested_by of [undefined, null, 'alice']) {

tests/unit/provisioning-refresh.test.ts

@@ -23,6 +23,15 @@ function deferred() {
 }
 
 describe('provisioning detail refresh ownership', () => {
+    it('requests a rebuild retry and refreshes the original failed operation', async () => {
+        const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true, deploymentId: 'recovery-a' })).mockResolvedValueOnce(response('a', 2));
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'));
+        await page.act('retry-rebuild', 'Follow the linked recovery operation.');
+        expect(fetcher).toHaveBeenNthCalledWith(1, '/api/provisioning/a/retry-rebuild', expect.objectContaining({ method: 'POST' }));
+        expect(fetcher).toHaveBeenNthCalledWith(2, '/api/provisioning/a?events=recent', expect.anything());
+        expect(page.state()).toMatchObject({ id: 'a', detail: detail('a', 2), notice: 'Follow the linked recovery operation.', busy: null });
+    });
     it('sends explicit override confirmation and refreshes the operation after acceptance', async () => {
         const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true })).mockResolvedValueOnce(response('a', 2));
         const page = setup(fetcher);