feat: add Watchdog recovery and drift rebuild workflows

src/lib/server/db/index.ts

@@ -16,6 +16,7 @@ let _db: Database.Database | null = null;
 // backfill columns; this can.
 const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
     { name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
+    { name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
     // Base columns that shipped in the very first schema stay here for parity
     // with fresh installs — no ALTER is issued if they're already present.
     { name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' },
@@ -246,6 +247,44 @@ function migrateProvisioningActions(handle: Database.Database): void {
     } finally { handle.pragma('foreign_keys = ON'); }
 }
 
+/** Rebuild constrained tables while preserving links, indexes, triggers, and row order. */
+function migrateWatchdogConstraints(handle: Database.Database): void {
+    for (const name of ['provisioning_sources', 'deployment_servers']) {
+        const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
+        const replacement = name === 'provisioning_sources'
+            ? (sql.includes("'watchdog'") ? sql : sql.replace("'campaign','beta','maintenance'", "'campaign','beta','maintenance','watchdog'"))
+            : sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
+        if (sql === replacement) continue;
+        const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
+            .all(name) as { sql: string }[];
+        handle.pragma('foreign_keys = OFF');
+        try {
+            handle.transaction(() => {
+                handle.exec(replacement.replace(name, name + '_watchdog_new'));
+                handle.exec(`INSERT INTO ${name}_watchdog_new (rowid, ${
+                    (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
+                }) SELECT rowid, ${
+                    (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
+                } FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};`);
+                for (const object of dependent) handle.exec(object.sql);
+                if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Watchdog migration found broken references');
+            })();
+        } finally { handle.pragma('foreign_keys = ON'); }
+    }
+}
+
+function backfillDesiredState(handle: Database.Database): void {
+    if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) return;
+    handle.transaction(() => {
+        // A failed stop may have reached Komodo. Preserve that explicit intent.
+        handle.exec(`UPDATE servers SET desired_state = 'stopped' WHERE 'stop' = (
+            SELECT action FROM server_deployments d WHERE d.server = servers.name AND d.state != 'cancelled'
+            AND d.action IN ('create','start','restart','rebuild','stop')
+            ORDER BY d.rowid DESC LIMIT 1);
+            INSERT INTO settings (key, value) VALUES ('watchdog_desired_state_v1', '1');`);
+    })();
+}
+
 export function db(): Database.Database {
     if (_db) return _db;
 
@@ -260,6 +299,7 @@ export function db(): Database.Database {
         dropLegacyBetaTables(handle);
         handle.exec(schemaSql);
         migrateProvisioningActions(handle);
+        migrateWatchdogConstraints(handle);
         migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
         migrateTable(handle, 'server_deployments', [
             { name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
@@ -270,6 +310,7 @@ export function db(): Database.Database {
             migrateTable(handle, table, [{ name: 'provisioning_id', ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` }]);
         }
         migrateServers(handle);
+        backfillDesiredState(handle);
         // Column additions must run BEFORE any statement that references the
         // new columns. schema.sql cannot include those references — see the
         // comment there.

src/lib/server/db/schema.sql

@@ -64,6 +64,7 @@ CREATE TABLE IF NOT EXISTS servers (
     komodo_name      TEXT NOT NULL,
     komodo_server    TEXT,
     container_state TEXT,                                -- last observed Komodo runtime state
+    desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped')),
     game_port        INTEGER,                             -- UDP RCON port; lazy-filled from Komodo
     rcon_host        TEXT,                                -- optional per-server override of RCON_HOST
     game             TEXT,                                -- 't5' etc.; static per-server (no map rotation)
@@ -299,7 +300,7 @@ CREATE INDEX IF NOT EXISTS idx_deployments_started ON deployments(started_at DES
 
 CREATE TABLE IF NOT EXISTS deployment_servers (
     deployment_id TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE,
-    server        TEXT NOT NULL REFERENCES servers(name),
+    server        TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.
     files_changed INTEGER NOT NULL,
     restart_state TEXT NOT NULL,     -- pending | done | failed | manual | held
     players       INTEGER,
@@ -445,7 +446,7 @@ CREATE TABLE IF NOT EXISTS server_rebuild_jobs (
 -- history retention, while current pointers live on the caller's server rows.
 CREATE TABLE IF NOT EXISTS provisioning_sources (
     operation_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
-    kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance')),
+    kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance','watchdog')),
     source_id TEXT NOT NULL,
     server TEXT NOT NULL,
     PRIMARY KEY (operation_id, kind, source_id, server)
@@ -622,3 +623,36 @@ CREATE TABLE IF NOT EXISTS metric_samples (
     capacity INTEGER,
     PRIMARY KEY (at, server)
 );
+
+-- Watchdog history deliberately survives server destruction/rebuild.
+CREATE TABLE IF NOT EXISTS watchdog_incidents (
+    id TEXT PRIMARY KEY,
+    server TEXT NOT NULL,
+    kind TEXT NOT NULL CHECK (kind IN ('health','drift')),
+    state TEXT NOT NULL CHECK (state IN ('open','recovering','attention','resolved','dismissed')),
+    summary TEXT NOT NULL,
+    details TEXT NOT NULL DEFAULT '{}',
+    first_seen_at TEXT NOT NULL,
+    last_seen_at TEXT NOT NULL,
+    finished_at TEXT,
+    failures INTEGER NOT NULL DEFAULT 1,
+    cycle INTEGER NOT NULL DEFAULT 1
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_watchdog_active ON watchdog_incidents(server,kind) WHERE state != 'resolved';
+CREATE TABLE IF NOT EXISTS watchdog_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    incident_id TEXT NOT NULL REFERENCES watchdog_incidents(id) ON DELETE CASCADE,
+    created_at TEXT NOT NULL,
+    message TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_watchdog_events ON watchdog_events(incident_id,id);
+CREATE TABLE IF NOT EXISTS watchdog_actions (
+    id TEXT PRIMARY KEY,
+    incident_id TEXT NOT NULL REFERENCES watchdog_incidents(id) ON DELETE CASCADE,
+    action TEXT NOT NULL CHECK (action IN ('restart','rebuild')),
+    operation_id TEXT NOT NULL UNIQUE,
+    campaign_id TEXT,
+    created_at TEXT NOT NULL,
+    cycle INTEGER NOT NULL DEFAULT 1,
+    UNIQUE(incident_id,cycle,action)
+);