Retry incomplete controller startup (Refs #65)

src/hooks.server.ts

@@ -9,21 +9,24 @@ import { withOperationSource } from '$lib/server/hosts/operation-context.js';
 import { withHostRequestContext } from '$lib/server/hosts/request-context.js';
 
 let booted = false;
+let rejectionLoggerRegistered = false;
 function boot(): void {
     if (booted) return;
-    booted = true;
+    if (!rejectionLoggerRegistered) {
-    process.on('unhandledRejection', reason => {
+        process.on('unhandledRejection', reason => {
-        console.error('[process] unhandled rejection:', reason);
+            console.error('[process] unhandled rejection:', reason);
-    });
+        });
+        rejectionLoggerRegistered = true;
+    }
     migrate();
     interruptDeployments();
     startWorker();
     startPerformanceMonitor();
+    booted = true;
 }
 
-// SvelteKit calls handle() for every request; using the first request as the
+// Boot synchronously before serving requests. Failed startup stays unbooted so
-// boot trigger keeps the whole app single-process without needing an
+// the next request retries migrations and worker startup before reaching the UI.
-// out-of-band init hook.
 export const handle: Handle = async (input) => {
     boot();
     const started = performance.now();

tests/unit/hooks-server.test.ts

@@ -10,7 +10,7 @@ vi.mock('../../src/lib/server/performance.js', () => ({ startPerformanceMonitor:
 
 let originalListeners = process.listeners('unhandledRejection');
 beforeEach(() => {
-    vi.resetModules(); vi.clearAllMocks();
+    vi.resetModules(); vi.resetAllMocks();
     originalListeners = process.listeners('unhandledRejection');
 });
 afterEach(() => {
@@ -21,6 +21,49 @@ afterEach(() => {
 });
 
 describe('controller error handling', () => {
+    it.each(['migration', 'deployment recovery', 'worker startup', 'performance monitor'] as const)(
+        'retries failed %s before serving requests and boots only once after recovery', async phase => {
+            const { handle } = await import('../../src/hooks.server.js');
+            const { migrate } = await import('../../src/lib/server/db/migrate.js');
+            const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js');
+            const { startWorker } = await import('../../src/lib/server/pipeline/worker.js');
+            const { startPerformanceMonitor } = await import('../../src/lib/server/performance.js');
+            const { authenticationHandle } = await import('../../src/lib/server/auth/guard.js');
+            const steps = [migrate, interruptDeployments, startWorker, startPerformanceMonitor];
+            const failedStep = ['migration', 'deployment recovery', 'worker startup', 'performance monitor'].indexOf(phase);
+            const failure = new Error('database or disk is full');
+            vi.mocked(steps[failedStep])
+                .mockImplementationOnce(() => { throw failure; })
+                .mockImplementationOnce(() => { throw failure; });
+            const input = {
+                event: { request: new Request('http://localhost/'), route: { id: '/' }, locals: {} },
+                resolve: vi.fn()
+            } as unknown as Parameters<Handle>[0];
+
+            for (let attempt = 1; attempt <= 2; attempt++) {
+                await expect(handle(input)).rejects.toBe(failure);
+                expect(authenticationHandle).not.toHaveBeenCalled();
+                for (const [index, step] of steps.entries()) {
+                    expect(step).toHaveBeenCalledTimes(index <= failedStep ? attempt : 0);
+                }
+            }
+
+            expect((await handle(input)).status).toBe(200);
+            expect((await handle(input)).status).toBe(200);
+            expect(authenticationHandle).toHaveBeenCalledTimes(2);
+            for (const [index, step] of steps.entries()) {
+                expect(step).toHaveBeenCalledTimes(index <= failedStep ? 3 : 1);
+                const lastCall = vi.mocked(step).mock.invocationCallOrder.at(-1)!;
+                expect(lastCall).toBeLessThan(vi.mocked(authenticationHandle).mock.invocationCallOrder[0]);
+                if (index > 0) {
+                    expect(lastCall).toBeGreaterThan(vi.mocked(steps[index - 1]).mock.invocationCallOrder.at(-1)!);
+                }
+            }
+            const listeners = process.listeners('unhandledRejection').filter(listener => !originalListeners.includes(listener));
+            expect(listeners).toHaveLength(1);
+        }
+    );
+
     it('registers one process rejection logger before starting workers, even across requests', async () => {
         const on = vi.spyOn(process, 'on');
         const log = vi.spyOn(console, 'error').mockImplementation(() => {});