feat: add Watchdog recovery and drift rebuild workflows

src/lib/server/db/index.ts

@@ -16,6 +16,7 @@ let _db: Database.Database | null = null;
 // backfill columns; this can.
 const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
     { name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
+    { name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
     // Base columns that shipped in the very first schema stay here for parity
     // with fresh installs — no ALTER is issued if they're already present.
     { name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' },
@@ -246,6 +247,44 @@ function migrateProvisioningActions(handle: Database.Database): void {
     } finally { handle.pragma('foreign_keys = ON'); }
 }
 
+/** Rebuild constrained tables while preserving links, indexes, triggers, and row order. */
+function migrateWatchdogConstraints(handle: Database.Database): void {
+    for (const name of ['provisioning_sources', 'deployment_servers']) {
+        const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
+        const replacement = name === 'provisioning_sources'
+            ? (sql.includes("'watchdog'") ? sql : sql.replace("'campaign','beta','maintenance'", "'campaign','beta','maintenance','watchdog'"))
+            : sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
+        if (sql === replacement) continue;
+        const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
+            .all(name) as { sql: string }[];
+        handle.pragma('foreign_keys = OFF');
+        try {
+            handle.transaction(() => {
+                handle.exec(replacement.replace(name, name + '_watchdog_new'));
+                handle.exec(`INSERT INTO ${name}_watchdog_new (rowid, ${
+                    (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
+                }) SELECT rowid, ${
+                    (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
+                } FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};`);
+                for (const object of dependent) handle.exec(object.sql);
+                if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Watchdog migration found broken references');
+            })();
+        } finally { handle.pragma('foreign_keys = ON'); }
+    }
+}
+
+function backfillDesiredState(handle: Database.Database): void {
+    if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) return;
+    handle.transaction(() => {
+        // A failed stop may have reached Komodo. Preserve that explicit intent.
+        handle.exec(`UPDATE servers SET desired_state = 'stopped' WHERE 'stop' = (
+            SELECT action FROM server_deployments d WHERE d.server = servers.name AND d.state != 'cancelled'
+            AND d.action IN ('create','start','restart','rebuild','stop')
+            ORDER BY d.rowid DESC LIMIT 1);
+            INSERT INTO settings (key, value) VALUES ('watchdog_desired_state_v1', '1');`);
+    })();
+}
+
 export function db(): Database.Database {
     if (_db) return _db;
 
@@ -260,6 +299,7 @@ export function db(): Database.Database {
         dropLegacyBetaTables(handle);
         handle.exec(schemaSql);
         migrateProvisioningActions(handle);
+        migrateWatchdogConstraints(handle);
         migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
         migrateTable(handle, 'server_deployments', [
             { name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
@@ -270,6 +310,7 @@ export function db(): Database.Database {
             migrateTable(handle, table, [{ name: 'provisioning_id', ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` }]);
         }
         migrateServers(handle);
+        backfillDesiredState(handle);
         // Column additions must run BEFORE any statement that references the
         // new columns. schema.sql cannot include those references — see the
         // comment there.