Rebuild missing containers during maintenance recovery

agent/ec_agent.py

@@ -33,7 +33,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.8.1"
+VERSION = "1.8.2"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1913,7 +1913,9 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 if kind == "instance.rebuild":
                     result = self.create(instance_id, next_generation, spec)
-                    shutil.rmtree(self.instance_dir(instance_id, generation), ignore_errors=False)
+                    old = self.instance_dir(instance_id, generation)
+                    if old.exists():
+                        shutil.rmtree(old, ignore_errors=False)
                     return result
                 directory = self.instance_dir(instance_id, generation)
                 if directory.exists():

agent/tests/test_agent.py

@@ -1444,6 +1444,44 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(self.agent.execute(self.operation("instance.stop", op_id="old-stop"))["state"], "failed")
         self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
 
+    def assert_missing_instance_rebuild(self, remove_directory=False):
+        payload = self.create()
+        self.docker.containers.pop(self.agent.container_name(INSTANCE, 1))
+        old = self.agent.instance_dir(INSTANCE, 1)
+        if remove_directory:
+            module.shutil.rmtree(old)
+        operation = self.operation("instance.rebuild", {"spec": payload, "nextGeneration": 2})
+        with patch.object(self.agent, "rcon", side_effect=AssertionError("Missing container queried player status")):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(json.loads(result["result"])["generation"], 2)
+        self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
+        self.assertFalse(old.exists())
+        self.assertEqual(json.loads((old.parent / "current.json").read_text()), {"generation": 2})
+        self.assertFalse(any(call[:2] in (("container", "stop"), ("container", "rm")) for call in self.docker.calls))
+        calls = list(self.docker.calls)
+        self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertEqual(self.docker.calls, calls)
+
+    def test_rebuild_recreates_missing_container(self):
+        self.assert_missing_instance_rebuild()
+
+    def test_rebuild_completes_when_old_instance_directory_is_also_missing(self):
+        self.assert_missing_instance_rebuild(remove_directory=True)
+
+    def test_rebuild_keeps_real_old_directory_cleanup_failures_unresolved(self):
+        payload = self.create()
+        self.docker.containers.pop(self.agent.container_name(INSTANCE, 1))
+        old = self.agent.instance_dir(INSTANCE, 1)
+        operation = self.operation("instance.rebuild", {"spec": payload, "nextGeneration": 2})
+        with patch.object(module.shutil, "rmtree", side_effect=PermissionError("Cleanup denied")):
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+            self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
+            self.assertTrue(old.exists())
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+        self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertFalse(old.exists())
+
     def test_traversal_and_symlinks_are_rejected(self):
         self.create()
         app = self.agent.instance_dir(INSTANCE, 1) / "app"

docs/guides/maintenance.md

@@ -127,15 +127,21 @@ an application restart, routine watchdog checks wait for a new status-monitor pa
 **Check now** and recovery verification still request fresh evidence, and
 Provisioning retains its checks before executing recovery actions.
 
-Three observations over at least two minutes are required before an automatic
+Three observations over at least two minutes are required before automatic
-health restart; starting or restarting containers get five minutes. Unknown host-agent responses are observations,
+health recovery; starting or restarting containers get five minutes. A confirmed
-not proof of a stopped container. Repeated manual checks do not accelerate this
+missing container goes directly to rebuild after this same sustained detection
+window. It cannot contain players, so recovery skips the player-count check.
+Offline hosts, unknown container state, and missing or stale observations wait
+for fresh evidence. Repeated manual checks do not accelerate this
 window. A successful routine or manual game-status probe between scans resets
 the failure streak. Missing RCON configuration, unresolved ports, and invalid
-replies remain diagnostic findings and cannot authorize health recovery.
+replies alone remain diagnostic findings and cannot authorize health recovery.
 
-A health finding gets one restart, then one rebuild if recovery fails after five
+A health finding for an existing container gets one restart, then one rebuild if
-minutes and fresh health checks still justify recovery. If current container and
+recovery fails after five minutes and fresh health checks still justify recovery.
+Missing-container recovery gets one direct rebuild. Both paths respect maintenance
+policy and pause settings, intentionally stopped servers, capacity limits, and
+protected beta or campaign work. If current container and
 game checks succeed before an automatic health restart or rebuild is submitted,
 the obsolete operation is cancelled and the finding is resolved. Unknown health
 holds recovery for another check. Drift repairs and requested lifecycle operations

src/lib/server/maintenance/recovery.ts

@@ -247,7 +247,7 @@ async function observe(server: string, operationId?: string, fresh = false): Pro
     const identity = snapshot?.identity ?? monitoringIdentity(server);
     const containerEvidence = { identity, observedAt: inspected.observedAt };
     const state = health.state?.toLowerCase() ?? null;
-    if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
+    if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['missing','stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
     const telemetry = telemetryEvidence(server);
     if (telemetry) return { identity, observedAt: telemetry.receivedAt, healthy: true, actionable: false, reason: 'Container healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
     const probed = snapshot ? snapshot.game : await rconForMonitoring(server, true).catch(error => {
@@ -280,16 +280,21 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
         const startup = ['starting','restarting'].includes(observation.containerState ?? '');
         const oldEnough = Date.parse(observation.observedAt) - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
         if (problem.state === 'open' && observation.actionable && oldEnough && problem.failures >= 3 && maintenanceAllowed('health')) {
+            // A missing container has nothing to restart. Recreate it through
+            // the normal rebuild queue, with its identity and safety checks.
+            const action = observation.containerState === 'missing' ? 'rebuild' : 'restart';
             const result = db().transaction(() => {
-                const next = requestWatchdogLifecycle(server, 'restart', problem.id);
+                const next = requestWatchdogLifecycle(server, action, problem.id);
-                if (next.ok) addAction(problem, next.deploymentId, 'restart');
+                if (next.ok) addAction(problem, next.deploymentId, action);
                 return next;
             })();
             if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error);
         }
     }
+    // Missing-container recovery belongs to the health policy. Do not turn
+    // absent files into drift or bypass the sustained health-failure window.
     // Unknown container health is an observation failure, not proof that a rebuild is safe.
-    if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
+    if (observation.containerState === 'missing' || (!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
     // File trees change far less often than liveness. Persist the last successful
     // scan per registration/process so health checks do not rescan every minute.
     // Manual checks and new lifecycle submissions always get fresh file evidence.

src/routes/maintenance/+page.svelte

@@ -53,7 +53,7 @@
     let checkedAt = $state<number | null>(null);
     const tabs = [{ id: 'overview', label: 'Overview' }, { id: 'policies', label: 'Policies & schedules' }, { id: 'activity', label: 'Activity' }];
     const policies: { id: MaintenancePolicy; title: string; description: string }[] = [
-        { id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Rebuilding deletes instance files and installs the latest promoted setup.' },
+        { id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Confirmed missing containers go directly to rebuild. Rebuilding deletes instance files and installs the latest promoted setup.' },
         { id: 'drift', title: 'Configuration drift', description: 'Rebuild servers whose managed files differ from the promoted setup. Record the differences in a linked Campaign. Rebuilding deletes instance files.' },
         { id: 'updates', title: 'Automatic Plutonium updates', description: 'Update outdated running servers on restart. One automatic attempt per published revision; failed verification requires inspection. Disabled by default.' }
     ];

src/routes/maintenance/[id]/+page.svelte

@@ -44,7 +44,8 @@
         if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.';
         if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.';
         if (incident.state === 'recovering') return 'Recovery is in progress. Follow the provisioning operations below for eligibility checks, progress, and health verification.';
-        return 'Maintenance detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server.';
+        if (incident.kind === 'health' && evidence.containerState?.toLowerCase() === 'missing') return 'Maintenance confirmed that the container is missing. When health recovery is enabled and Maintenance is not paused, three observations over at least two minutes trigger a direct rebuild without a player-count check. Capacity limits and protected work still apply.';
+        return 'Maintenance detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server. Offline hosts and unknown container states wait for another check.';
     });
 
     async function load() {
@@ -114,7 +115,7 @@
             <h2>{incident.summary}</h2>
             <p>{explanation}</p>
             {#if activeOperation && (canRetry || canDismiss)}<p>Complete or cancel the active provisioning operation before retrying or dismissing this incident.</p>{/if}
-            {#if !incident.server_deleted_at}{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}{/if}
+            {#if !incident.server_deleted_at}{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery restarts an existing container, then rebuilds if needed. A confirmed missing container goes directly to rebuild. Rebuilding deletes any remaining instance files and installs the latest promoted setup.</p>{/if}{/if}
             <dl>
                 <div><dt>First detected</dt><dd>{$timeFormat.dateTime(incident.first_seen_at)}</dd></div>
                 <div><dt>Last observed</dt><dd>{$timeFormat.dateTime(incident.last_seen_at)}</dd></div>

static/agent/ec-agent.py

@@ -33,7 +33,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.8.1"
+VERSION = "1.8.2"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1913,7 +1913,9 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 if kind == "instance.rebuild":
                     result = self.create(instance_id, next_generation, spec)
-                    shutil.rmtree(self.instance_dir(instance_id, generation), ignore_errors=False)
+                    old = self.instance_dir(instance_id, generation)
+                    if old.exists():
+                        shutil.rmtree(old, ignore_errors=False)
                     return result
                 directory = self.instance_dir(instance_id, generation)
                 if directory.exists():

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.8.1",
+  "version": "1.8.2",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "09c722864c73ddb76404ea994d9a194892a2226d8d9006de8dcac9a6e2591760",
+  "sha256": "9715027ee71b4bab7df3ec44186eb81554f913741f1dde3e5d79b7e525a32ba6",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/watchdog.test.ts

@@ -51,6 +51,7 @@ const onlyIncident = () => { expect(incidents()).toHaveLength(1); return inciden
 const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[];
 function advance(ms: number) { vi.setSystemTime(Date.now() + ms); refreshHost(); }
 function down() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
+function missing() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'missing', reason: 'Container missing' }); }
 async function pollStatus() {
     const worker = await vi.importActual<typeof import('../../src/lib/server/pipeline/worker.js')>('../../src/lib/server/pipeline/worker.js');
     await worker.fanoutStatus();
@@ -95,6 +96,98 @@ describe('Watchdog detection and ownership', () => {
         expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0);
     });
 
+    it.each([false, true])('rebuilds a confirmed missing container once after three samples over two minutes (manual=%s)', async manual => {
+        missing();
+        const drift = await import('../../src/lib/server/fleet/drift.js');
+        const scan = vi.spyOn(drift, 'prepareDriftCampaign');
+        for (let sample = 0; sample < 3; sample++) {
+            if (!manual) await pollStatus();
+            await watchdog.watchdogTick(manual);
+            expect(onlyIncident().failures).toBe(sample + 1);
+            if (sample < 2) {
+                expect(operations()).toEqual([]);
+                advance(watchdog.WATCHDOG_SCAN_MS);
+            }
+        }
+        const incident = onlyIncident();
+        const queued = operations();
+        expect(incident).toMatchObject({ kind: 'health', state: 'recovering', failures: 3 });
+        expect(queued).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
+        expect(watchdog.watchdogDetail(incident.id)?.actions).toEqual([
+            expect.objectContaining({ action: 'rebuild', operation_id: queued[0].id, campaign_id: null })
+        ]);
+        expect(database.db().prepare('SELECT kind,source_id FROM provisioning_sources WHERE operation_id=?').all(queued[0].id))
+            .toEqual([{ kind: 'watchdog', source_id: incident.id }]);
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        expect(scan).not.toHaveBeenCalled();
+        expect(backend.nativeRequest).not.toHaveBeenCalled();
+        database.closeDb();
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual(queued);
+        expect(watchdog.watchdogDetail(incident.id)?.actions).toHaveLength(1);
+    });
+
+    it('does not accelerate missing-container recovery through repeated manual checks', async () => {
+        missing();
+        await watchdog.watchdogTick(true);
+        for (let check = 0; check < 5; check++) await watchdog.watchdogTick(true);
+        expect(onlyIncident().failures).toBe(1);
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        advance(watchdog.WATCHDOG_SCAN_MS - 1); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([]);
+        advance(1); await watchdog.watchdogTick(true);
+        expect(onlyIncident().failures).toBe(3);
+        expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
+    });
+
+    it('dispatches missing-container recovery as a rebuild without requesting an empty-player count', async () => {
+        missing();
+        for (let sample = 0; sample < 3; sample++) {
+            await watchdog.watchdogTick(true);
+            if (sample < 2) advance(watchdog.WATCHDOG_SCAN_MS);
+        }
+        const operation = operations()[0];
+        expect(operation).toMatchObject({ action: 'rebuild', state: 'waiting' });
+        const readFiles = vi.mocked(backend.nativeRequest).getMockImplementation()!;
+        vi.mocked(backend.nativeRequest).mockImplementation(async (instance, kind, payload) => {
+            if (kind === 'instance.inspect') return { exists: false, healthy: false, state: 'missing' } as never;
+            return readFiles(instance, kind, payload);
+        });
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        const binding = database.db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(operation.id) as { operation_id: string | null };
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        expect(binding.operation_id).toEqual(expect.any(String));
+        expect(getHostOperation(binding.operation_id!)).toMatchObject({ kind: 'instance.rebuild', generation: 1,
+            payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
+        expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]);
+        expect(rcon.rconGetPlayerCount).not.toHaveBeenCalled();
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+    });
+
+    it.each(['health disabled', 'maintenance paused'])('keeps missing containers observational with %s even when files have drifted', async policy => {
+        missing();
+        const policies = await import('../../src/lib/server/maintenance/policies.js');
+        policies.setMaintenancePolicy('drift', true);
+        if (policy === 'health disabled') policies.setMaintenancePolicy('health', false);
+        else policies.pauseMaintenance(true);
+        await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
+        const drift = await import('../../src/lib/server/fleet/drift.js');
+        const scan = vi.spyOn(drift, 'prepareDriftCampaign');
+        for (let sample = 0; sample < 4; sample++) {
+            await watchdog.watchdogTick(true);
+            advance(watchdog.WATCHDOG_SCAN_MS);
+        }
+        expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 4 });
+        expect(operations()).toEqual([]);
+        expect(scan).not.toHaveBeenCalled();
+        expect(backend.nativeRequest).not.toHaveBeenCalled();
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        policies.setMaintenancePolicy('health', true); policies.pauseMaintenance(false);
+        await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
+    });
+
     it('does not count the same sample again or use stale data, and resets after a monitoring gap', async () => {
         down();
         await pollStatus();
@@ -257,8 +350,12 @@ describe('Watchdog detection and ownership', () => {
         expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]);
     });
 
-    it('does not restart an unknown host agent state even after repeated sustained observations', async () => {
+    it.each([
-        vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, reason: 'host agent request timed out' });
+        ['unavailable', undefined, 'host agent request timed out'],
+        ['unknown', 'unknown', 'Agent could not inspect the container'],
+        ['offline', 'unknown', 'Host agent is disconnected']
+    ] as const)('does not recover an %s host agent state even after repeated sustained observations', async (_label, state, reason) => {
+        vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state, reason });
         for (let n = 0; n < 8; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
         expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' });
         expect(operations()).toEqual([]);
@@ -323,13 +420,13 @@ describe('Watchdog detection and ownership', () => {
         expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
     });
 
-    it('preserves intentional stop intent after completion and database reopening', async () => {
+    it.each(['exited', 'missing'])('preserves intentional stop intent for %s containers after completion and database reopening', async state => {
         const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
         const stop = requestLifecycle(server, 'stop');
         expect(stop.ok).toBe(true);
         if (!stop.ok) throw new Error(stop.error);
         database.db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ? WHERE id = ?").run(new Date().toISOString(), stop.deploymentId);
-        down();
+        if (state === 'missing') missing(); else down();
         database.closeDb();
         for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
         expect(incidents()).toEqual([]);
@@ -354,11 +451,11 @@ describe('Watchdog detection and ownership', () => {
         expect(backend.nativeHealth).not.toHaveBeenCalled();
     });
 
-    it('rechecks lifecycle ownership after a health probe yields to an operator stop', async () => {
+    it.each(['exited', 'missing'])('rechecks lifecycle ownership after a %s probe yields to an operator stop', async state => {
         const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
         vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
             expect(requestLifecycle(server, 'stop').ok).toBe(true);
-            return { ok: false, state: 'exited', reason: 'Container exited' };
+            return { ok: false, state, reason: `Container ${state}` };
         });
         await watchdog.watchdogTick(true);
         expect(incidents()).toEqual([]);
@@ -412,6 +509,31 @@ describe('Watchdog detection and ownership', () => {
 
 describe('Watchdog recovery persistence', () => {
 
+    it('escalates a submitted restart that becomes missing after its five-minute recovery window', async () => {
+        const incident = await sustainedFailure();
+        const restart = operations()[0];
+        const submittedAt = new Date().toISOString();
+        database.db().prepare("UPDATE server_deployments SET state='awaiting-health',submitted_at=? WHERE id=?")
+            .run(submittedAt, restart.id);
+        database.db().prepare('INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES(?,?,?)')
+            .run(server, restart.id, submittedAt);
+        missing();
+        vi.mocked(rcon.rconStatusResult).mockClear();
+        advance(5 * 60_000 - 1); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([{ ...restart, state: 'awaiting-health' }]);
+        advance(1); await watchdog.watchdogTick(true);
+        const rebuilt = operations();
+        expect(rebuilt).toEqual([
+            { ...restart, state: 'failed' }, expect.objectContaining({ action: 'rebuild', state: 'waiting' })
+        ]);
+        expect(watchdog.watchdogDetail(incident.id)?.actions.map(action => action.action)).toEqual(['restart', 'rebuild']);
+        expect(database.db().prepare('SELECT operation_id FROM restart_rollout WHERE server=?').get(server))
+            .toEqual({ operation_id: rebuilt[1].id });
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual(rebuilt);
+    });
+
     it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
         down();
         await watchdog.watchdogTick(true);