feat: add Watchdog recovery and drift rebuild workflows

src/routes/api/campaigns/[id]/+server.ts

@@ -39,5 +39,6 @@ export async function GET({ params }: { params: { id: string } }) {
         .all(params.id);
 
     const events = db().prepare('SELECT id, created_at, message FROM campaign_events WHERE deployment_id = ? ORDER BY id').all(params.id);
-    return json({ deployment: dep, servers, files, scripts, events });
+    const watchdog = db().prepare('SELECT incident_id FROM watchdog_actions WHERE campaign_id = ? LIMIT 1').get(params.id) as { incident_id: string } | undefined;
+    return json({ deployment: dep, servers, files, scripts, events, watchdogIncidentId: watchdog?.incident_id ?? null });
 }

src/routes/api/drift/+server.ts

@@ -1,6 +1,5 @@
 import { json } from '@sveltejs/kit';
-import { DriftPlanChanged, previewDrift, runDriftDeployment } from '$lib/server/fleet/drift.js';
-import { kick } from '$lib/server/pipeline/worker.js';
+import { previewDrift } from '$lib/server/fleet/drift.js';
 
 // GET returns the current drift plan for the whole fleet without touching disk.
 export async function GET() {
@@ -27,7 +26,7 @@ export async function GET() {
     });
 }
 
-// POST removes drift and creates a drift deployment. Body: { servers?: string[], dryRun?: boolean }
+// Legacy previews remain readable. Watchdog owns all new reconciliation requests.
 export async function POST({ request }: { request: Request }) {
     const body = await request.json().catch(() => null);
     if (!body || (body.servers !== undefined && (!Array.isArray(body.servers) || !body.servers.every((server: unknown) => typeof server === 'string')))) {
@@ -35,12 +34,8 @@ export async function POST({ request }: { request: Request }) {
     }
     try {
         if (body.dryRun === true) return json({ ok: true, plan: await previewDrift(body.servers) });
-        if (typeof body.expectedFingerprint !== 'string' || !body.expectedFingerprint) return json({ ok: false, error: 'Review a reconciliation plan before applying changes.' }, { status: 400 });
-        const result = await runDriftDeployment({ servers: body.servers, expectedFingerprint: body.expectedFingerprint, triggeredBy: 'ui' });
-        if (result.deploymentId) void kick();
-        return json({ ok: true, result });
+        return json({ ok: false, error: 'Drift reconciliation has moved to Watchdog. Scan there to queue a rebuild and campaign.', watchdogUrl: '/watchdog' }, { status: 410 });
     } catch (error) {
-        if (error instanceof DriftPlanChanged) return json({ ok: false, error: error.message, plan: error.plan }, { status: 409 });
         return json({ ok: false, error: (error as Error).message }, { status: 400 });
     }
 }

src/routes/api/watchdog/+server.ts

@@ -0,0 +1,18 @@
+import { json } from '@sveltejs/kit';
+import { watchdogOverview, watchdogTick, setWatchdogEnabled } from '$lib/server/watchdog/index.js';
+import { kick } from '$lib/server/pipeline/worker.js';
+
+export function GET() { return json(watchdogOverview()); }
+export async function POST({ request }: { request: Request }) {
+    const body = await request.json().catch(() => null);
+    if (!body || typeof body !== 'object' || Array.isArray(body)
+        || (body.enabled !== undefined && typeof body.enabled !== 'boolean')
+        || (body.action !== 'scan' && typeof body.enabled !== 'boolean')
+        || (body.action !== undefined && body.action !== 'scan')) {
+        return json({ error: 'Provide action: scan or an enabled boolean.' }, { status: 400 });
+    }
+    if (typeof body.enabled === 'boolean') setWatchdogEnabled(body.enabled);
+    if (body.action === 'scan') await watchdogTick(true);
+    void kick().catch(err => console.error('[watchdog] worker kick failed:', err));
+    return json(watchdogOverview());
+}

src/routes/api/watchdog/[id]/+server.ts

@@ -0,0 +1,16 @@
+import { json } from '@sveltejs/kit';
+import { watchdogDetail, updateWatchdogIncident } from '$lib/server/watchdog/index.js';
+import { kick } from '$lib/server/pipeline/worker.js';
+
+export function GET({ params }: { params: { id: string } }) {
+    const detail = watchdogDetail(params.id);
+    return detail ? json(detail) : json({ error: 'Watchdog incident not found.' }, { status: 404 });
+}
+export async function POST({ params, request }: { params: { id: string }; request: Request }) {
+    const body = await request.json().catch(() => null);
+    if (!body || !['retry','dismiss'].includes(body.action)) return json({ error: 'Choose retry or dismiss.' }, { status: 400 });
+    const result = updateWatchdogIncident(params.id, body.action);
+    if (!result.ok) return json({ error: result.error }, { status: result.status });
+    void kick().catch(err => console.error('[watchdog] worker kick failed:', err));
+    return json(watchdogDetail(params.id));
+}