feat: add Watchdog recovery and drift rebuild workflows

src/lib/server/db/schema.sql

@@ -64,6 +64,7 @@ CREATE TABLE IF NOT EXISTS servers (
     komodo_name      TEXT NOT NULL,
     komodo_server    TEXT,
     container_state TEXT,                                -- last observed Komodo runtime state
+    desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped')),
     game_port        INTEGER,                             -- UDP RCON port; lazy-filled from Komodo
     rcon_host        TEXT,                                -- optional per-server override of RCON_HOST
     game             TEXT,                                -- 't5' etc.; static per-server (no map rotation)
@@ -299,7 +300,7 @@ CREATE INDEX IF NOT EXISTS idx_deployments_started ON deployments(started_at DES
 
 CREATE TABLE IF NOT EXISTS deployment_servers (
     deployment_id TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE,
-    server        TEXT NOT NULL REFERENCES servers(name),
+    server        TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.
     files_changed INTEGER NOT NULL,
     restart_state TEXT NOT NULL,     -- pending | done | failed | manual | held
     players       INTEGER,
@@ -445,7 +446,7 @@ CREATE TABLE IF NOT EXISTS server_rebuild_jobs (
 -- history retention, while current pointers live on the caller's server rows.
 CREATE TABLE IF NOT EXISTS provisioning_sources (
     operation_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
-    kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance')),
+    kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance','watchdog')),
     source_id TEXT NOT NULL,
     server TEXT NOT NULL,
     PRIMARY KEY (operation_id, kind, source_id, server)
@@ -622,3 +623,36 @@ CREATE TABLE IF NOT EXISTS metric_samples (
     capacity INTEGER,
     PRIMARY KEY (at, server)
 );
+
+-- Watchdog history deliberately survives server destruction/rebuild.
+CREATE TABLE IF NOT EXISTS watchdog_incidents (
+    id TEXT PRIMARY KEY,
+    server TEXT NOT NULL,
+    kind TEXT NOT NULL CHECK (kind IN ('health','drift')),
+    state TEXT NOT NULL CHECK (state IN ('open','recovering','attention','resolved','dismissed')),
+    summary TEXT NOT NULL,
+    details TEXT NOT NULL DEFAULT '{}',
+    first_seen_at TEXT NOT NULL,
+    last_seen_at TEXT NOT NULL,
+    finished_at TEXT,
+    failures INTEGER NOT NULL DEFAULT 1,
+    cycle INTEGER NOT NULL DEFAULT 1
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_watchdog_active ON watchdog_incidents(server,kind) WHERE state != 'resolved';
+CREATE TABLE IF NOT EXISTS watchdog_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    incident_id TEXT NOT NULL REFERENCES watchdog_incidents(id) ON DELETE CASCADE,
+    created_at TEXT NOT NULL,
+    message TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_watchdog_events ON watchdog_events(incident_id,id);
+CREATE TABLE IF NOT EXISTS watchdog_actions (
+    id TEXT PRIMARY KEY,
+    incident_id TEXT NOT NULL REFERENCES watchdog_incidents(id) ON DELETE CASCADE,
+    action TEXT NOT NULL CHECK (action IN ('restart','rebuild')),
+    operation_id TEXT NOT NULL UNIQUE,
+    campaign_id TEXT,
+    created_at TEXT NOT NULL,
+    cycle INTEGER NOT NULL DEFAULT 1,
+    UNIQUE(incident_id,cycle,action)
+);