josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Suppress maintenance watchers during provisioning

Browse Source 
Keep maintenance observations and automatic requests clear of active lifecycle operations and retained recovery capacity while preserving owned recovery and explicit retries.

Fixes #55
This commit is contained in:
josh
2026-10-01 11:54:22 -04:00
parent a3cd3ee585
commit 0cb48c435e
8 changed files with 366 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/maintenance/runtime.ts
+12 -1
View File
@@ -1,5 +1,6 @@
import { db } from '../db/index.js';
import { getServer, listServers } from '../fleet/servers.js';
import { hasActiveProvisioning } from '../fleet/provisioning.js';
import { requestRestart, cancelProvisioning } from '../provisioning/index.js';
import { nextMaintenanceAt } from '../../plutonium.js';
import type { MaintenanceJob, MaintenanceSchedule, PlutoniumStatus } from '../../plutonium.js';
@@ -37,6 +38,7 @@ export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind']
let added = 0;
for (const server of new Set(servers)) {
if (getServer(server)?.desired_state === 'stopped') continue;
if (origin !== 'manual' && hasActiveProvisioning(server)) continue;
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes;
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')").get(server) as { id: number };
@@ -125,6 +127,9 @@ async function runTick(): Promise<void> {
async function processJob(job: MaintenanceJob): Promise<void> {
const server = getServer(job.server);
if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
if (!job.provisioning_id && !job.restarted_at && hasActiveProvisioning(job.server)) {
return hold(job, 'Waiting for another provisioning operation.');
}
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);
@@ -136,7 +141,13 @@ async function processJob(job: MaintenanceJob): Promise<void> {
}
return hold(job, operation.stage);
}
const revision = await probeVersion(job.server);
if (hasActiveProvisioning(job.server, operation.id)) {
return hold(job, 'Waiting for another provisioning operation before verifying the running revision.');
}
const revision = await probeVersion(job.server, { maintenance: true });
if (hasActiveProvisioning(job.server, operation.id)) {
return hold(job, 'Waiting for another provisioning operation before verifying the running revision.');
}
if (job.target_revision && (revision === null || revision < job.target_revision)) {
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
}