josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/worker.ts
+168 -43
View File
@@ -8,43 +8,110 @@ import type { RuntimeAction } from '../../provisioning.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { processNativeProvisioning } from './native.js';
interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string }
const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[];
const priority = (sources: Source[]) => sources.some(s => s.kind === 'watchdog') ? 2 : sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2;
interface Source {
kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog';
source_id: string;
}
const sourcesFor = (id: string) =>
db()
.prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?')
.all(id) as Source[];
const priority = (sources: Source[]) =>
sources.some((s) => s.kind === 'watchdog')
? 2
: sources.some((s) => s.kind === 'beta')
? 0
: sources.some((s) => s.kind === 'campaign')
? 1
: sources.some((s) => s.kind === 'maintenance')
? 3
: 2;
/** Source work may hold or withdraw a request, but never executes it. */
function sourceEligibility(operation: Operation, onHold?: (source: Source) => void): 'ready' | 'held' | 'cancelled' {
function sourceEligibility(
operation: Operation,
onHold?: (source: Source) => void
): 'ready' | 'held' | 'cancelled' {
const sources = sourcesFor(operation.id);
if (!sources.length) return 'ready';
// Watchdog intent gates every linked source, including its drift campaign.
for (const source of sources.filter(source => source.kind === 'watchdog')) {
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(source.source_id, operation.server)
|| getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
const incident = db().prepare('SELECT kind FROM watchdog_incidents WHERE id = ?').get(source.source_id) as { kind: 'health' | 'drift' };
if (!maintenanceAllowed(incident.kind)) { onHold?.(source); return 'held'; }
for (const source of sources.filter((source) => source.kind === 'watchdog')) {
if (
!db()
.prepare(
"SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')"
)
.get(source.source_id, operation.server) ||
getServer(operation.server)?.desired_state === 'stopped'
)
return 'cancelled';
const incident = db()
.prepare('SELECT kind FROM watchdog_incidents WHERE id = ?')
.get(source.source_id) as { kind: 'health' | 'drift' };
if (!maintenanceAllowed(incident.kind)) {
onHold?.(source);
return 'held';
}
}
let held = false;
let ready = false;
for (const source of sources) {
if (source.kind === 'campaign') {
const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL')
.get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined;
const row = db()
.prepare(
'SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL'
)
.get(source.source_id, operation.server) as
| { restart_state: string; files_changed: number; provisioning_id: string }
| undefined;
if (row?.provisioning_id !== operation.id || !row.files_changed) continue;
if (['pending','failed'].includes(row.restart_state)) ready = true;
if (['held','manual'].includes(row.restart_state)) { held = true; onHold?.(source); }
if (['pending', 'failed'].includes(row.restart_state)) ready = true;
if (['held', 'manual'].includes(row.restart_state)) {
held = true;
onHold?.(source);
}
} else if (source.kind === 'beta') {
const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`)
.get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined;
if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue;
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
if (!row.outcome || ['active','passed'].includes(row.outcome)) ready = true;
if (row.outcome === 'paused') { held = true; onHold?.(source); }
const row = db()
.prepare(
`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`
)
.get(source.source_id, operation.server) as
| { state: string; stage: string; provisioning_id: string; outcome: string | null }
| undefined;
if (
row?.provisioning_id !== operation.id ||
!['running', 'ready-promote'].includes(row.state)
)
continue;
if (!['awaiting-empty', 'restarting', 'awaiting-health'].includes(row.stage)) continue;
if (!row.outcome || ['active', 'passed'].includes(row.outcome)) ready = true;
if (row.outcome === 'paused') {
held = true;
onHold?.(source);
}
} else if (source.kind === 'maintenance') {
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) {
const row = db()
.prepare(
'SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL'
)
.get(source.source_id) as
| { state: string; provisioning_id: string; kind: string; origin: string }
| undefined;
if (
row?.provisioning_id === operation.id &&
['queued', 'restarting'].includes(row.state)
) {
if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) { onHold?.(source); return 'held'; }
if (
!maintenanceAllowed() ||
(row.kind === 'update' &&
row.origin === 'automatic' &&
!maintenanceAllowed('updates'))
) {
onHold?.(source);
return 'held';
}
ready = true;
}
} else ready = true;
@@ -55,56 +122,114 @@ function sourceEligibility(operation: Operation, onHold?: (source: Source) => vo
function blocker(server: string, id: string, action: RuntimeAction): ProvisioningBlocker | null {
const sources = sourcesFor(id);
const rank = priority(sources);
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[];
const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number };
const conflict = others.find(other => other.state !== 'waiting' || other.action === 'destroy'
|| priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid));
const others = db()
.prepare(
`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health')`
)
.all(server, id) as { id: string; state: string; action: string; rowid: number }[];
const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as {
rowid: number;
};
const conflict = others.find(
(other) =>
other.state !== 'waiting' ||
other.action === 'destroy' ||
priority(sourcesFor(other.id)) < rank ||
(priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid)
);
if (conflict) {
return { code: 'operation-conflict', message: 'Waiting for another provisioning operation on this server', automatic: true,
href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' };
return {
code: 'operation-conflict',
message: 'Waiting for another provisioning operation on this server',
automatic: true,
href: `/provisioning/${encodeURIComponent(conflict.id)}`,
label: 'Blocking operation'
};
}
// Beta activation must not wait on the campaign rows that the beta itself
// holds. Campaign/maintenance callers likewise never block on their own job.
const beta = rank > 0 ? db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server) as { id: string } | undefined : undefined;
if (beta) return { code: 'beta-work', message: 'Waiting for beta work on this server', automatic: true, href: `/betas/${encodeURIComponent(beta.id)}`, label: 'Blocking beta' };
const campaign = rank > 1 ? db().prepare(`SELECT ds.deployment_id FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
const beta =
rank > 0
? (db()
.prepare(
`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`
)
.get(server) as { id: string } | undefined)
: undefined;
if (beta)
return {
code: 'beta-work',
message: 'Waiting for beta work on this server',
automatic: true,
href: `/betas/${encodeURIComponent(beta.id)}`,
label: 'Blocking beta'
};
const campaign =
rank > 1
? (db()
.prepare(
`SELECT ds.deployment_id FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
LIMIT 1`).get(server, id) as { deployment_id: string } | undefined : undefined;
if (campaign) return { code: 'campaign-work', message: 'Waiting for campaign work on this server', automatic: true,
href: `/campaigns/${encodeURIComponent(campaign.deployment_id)}`, label: 'Blocking campaign' };
LIMIT 1`
)
.get(server, id) as { deployment_id: string } | undefined)
: undefined;
if (campaign)
return {
code: 'campaign-work',
message: 'Waiting for campaign work on this server',
automatic: true,
href: `/campaigns/${encodeURIComponent(campaign.deployment_id)}`,
label: 'Blocking campaign'
};
return rolloutBlocker(server, id, action);
}
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
interface Operation {
id: string;
server: string;
action: RuntimeAction;
state: string;
stage: string;
}
let awaitedPass: Promise<void> | null = null;
export function processProvisioning(options: { wait?: boolean } = {}): Promise<void> {
if (options.wait !== false && awaitedPass) return awaitedPass;
// Each job owns its flight, so slow observations do not block new work.
const pass = nativePass(options);
if (options.wait === false) return pass;
awaitedPass = pass.finally(() => { awaitedPass = null; });
awaitedPass = pass.finally(() => {
awaitedPass = null;
});
return awaitedPass;
}
export const processLifecycle = processProvisioning;
function nativePass(options: { wait?: boolean }): Promise<void> {
return processNativeProvisioning(operation => {
return processNativeProvisioning((operation) => {
const scaling = scalingAdmission(operation.id);
if (scaling) return scaling;
if (operation.action === 'create') return null;
if (operation.action === 'destroy') {
const other = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL
AND (state='running' OR (state='awaiting-health' AND action NOT IN ('create','rebuild'))) LIMIT 1`).get(operation.server, operation.id);
const other = db()
.prepare(
`SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL
AND (state='running' OR (state='awaiting-health' AND action NOT IN ('create','rebuild'))) LIMIT 1`
)
.get(operation.server, operation.id);
return other ? 'Waiting for the current server operation.' : null;
}
const typed = operation as Operation;
const eligibility = sourceEligibility(typed);
if (eligibility === 'cancelled') { cancelProvisioning(operation.id, 'Requester cancelled or replaced this operation'); return 'Requester cancelled this operation.'; }
if (eligibility === 'cancelled') {
cancelProvisioning(operation.id, 'Requester cancelled or replaced this operation');
return 'Requester cancelled this operation.';
}
if (eligibility === 'held') return 'Waiting for the requester to resume.';
return blocker(operation.server, operation.id, typed.action)?.message ?? null;
}, options);