josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Move fleet retention and concurrency controls into Settings

Browse Source 
Persist settings in SQLite, import legacy environment values once, and apply changes to cleanup and deployment queues without restarting.

Closes #51
This commit is contained in:
josh
2026-09-27 16:11:45 -04:00
parent fc46cb9da4
commit 4fe8613678
20 changed files with 334 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/instances.ts
+4 -3
View File
@@ -1,3 +1,4 @@
import { fleetSettings } from '../settings.js';
import { probePlayerCount, probeServerStatus } from '../fleet/status-evidence.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
import { randomUUID } from 'node:crypto';
@@ -258,7 +259,7 @@ export async function rebuildServer(id: string): Promise<void> {
const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
if (!job) return;
if (creating || destroying.size || creationQueue().active.length >= config.createConcurrency) {
if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) {
// The cleanup checkpoint must survive capacity waits and app restarts.
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
return;
@@ -775,7 +776,7 @@ async function runCreations(): Promise<CreationOutcomes> {
// Snapshot the budget once. Fast success/failure must not turn a single
// worker pass into an unbounded stream of expensive container startups.
const queue = creationQueue();
const available = Math.max(0, config.createConcurrency - queue.active.length);
const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length);
const eligible = queue.queued.filter(({ id }) => {
const blocker = creationBlocker(id);
if (blocker) deploymentStage(id, blocker);
@@ -785,7 +786,7 @@ async function runCreations(): Promise<CreationOutcomes> {
if (creating || destroying.size > 0) break;
// API calls can cancel a selected job or resume an older startup while
// a previous provision awaits Komodo. Recheck without expanding this pass.
if (creationQueue().active.length >= config.createConcurrency) break;
if (creationQueue().active.length >= fleetSettings().createConcurrency) break;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
const blocker = creationBlocker(id);
if (blocker) {
src/lib/server/provisioning/restarts.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { config } from '../config.js';
import { fleetSettings } from '../settings.js';
import { db } from '../db/index.js';
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
import { probeServerStatus } from '../fleet/status-evidence.js';
@@ -49,7 +49,7 @@ export function rolloutWaitReason(server: string, operationId = ''): string | nu
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
}
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
}
export type RolloutRestartResult =