josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add safe manual instance rebalancing

Browse Source
This commit is contained in:
josh
2026-10-08 12:37:56 -04:00
parent 26de43639f
commit 012cd51a66
45 changed files with 2006 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/history.ts
+15 -6
View File
@@ -11,7 +11,8 @@ import { readProgress, recordProgress, type ProgressUpdate } from './progress.js
import { bindNativeOperation } from './native.js';
import { workerActivity } from '../pipeline/activity.js';
import { canCancelCreation } from './cancel-creation.js';
import { canCancelProvisioningJob, provisioningCancelled } from './cancellation.js';
import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js';
import { committedRebalanceStartup } from './concurrency.js';
import { nativeDeployment } from '../backends/native.js';
import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
import { serverTitle } from '../../ui/server-labels.js';
@@ -23,7 +24,7 @@ export function beginDeployment(server: string, action: ProvisioningAction, trig
db().transaction(() => {
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by, requested_by)
VALUES (?, ?, ?, 'running', 'Starting', ?, ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy,
triggeredBy === 'ui' ? currentRequestUsername() : null);
triggeredBy === 'ui' || triggeredBy.startsWith('rebalance:') ? currentRequestUsername() : null);
bindNativeOperation(id, server);
recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested',
label: action === 'create' ? 'Server creation requested'
@@ -63,6 +64,11 @@ function warningMessages(raw: string): string[] {
function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
const id = source.source_id;
if (source.kind === 'rebalance') {
const exists = !!db().prepare('SELECT 1 FROM rebalancing_runs WHERE id=?').get(id);
return { ...source, label: exists ? 'Rebalance run' : 'Rebalance history expired',
href: exists ? `/capacity?rebalance=1&rebalanceRun=${encodeURIComponent(id)}` : null };
}
if (source.kind === 'autoscaling') {
const exists = !!db().prepare('SELECT 1 FROM autoscaling_policies WHERE id=?').get(id);
return { ...source, label: exists ? 'Autoscaling policy' : 'Deleted autoscaling policy', href: exists ? `/autoscaling#${encodeURIComponent(id)}` : null };
@@ -94,7 +100,7 @@ function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): P
const pools: ProvisioningCapacity['pools'] = [];
if (deployment.action === 'create' || deployment.action === 'rebuild') {
const jobs = db().prepare(`SELECT id, server, state FROM server_deployments
WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health')
WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup})
ORDER BY started_at, rowid`).all() as { id: string; server: string; state: string }[];
const holders = jobs.filter(job => job.state !== 'queued').map(({ id, server }) => ({ id, server }));
const order = jobs.filter(job => job.state === 'queued').findIndex(job => job.id === deployment.id);
@@ -127,6 +133,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
if (row.triggered_by.startsWith('rebalance:')) sources.push(sourceDetails({ kind: 'rebalance', source_id: row.triggered_by.slice('rebalance:'.length) }));
const launch = recipeSummaryForDeployment(id);
const collection = db().prepare(`SELECT c.id, c.name FROM (
SELECT collection_id, deployment_id FROM server_collection_members
@@ -271,7 +278,8 @@ const deploymentListColumns = `d.id, d.server, d.server_deleted_at, d.action, d.
CASE WHEN json_valid(d.observation_json) THEN json_extract(d.observation_json,'$.blocker.code') END AS waitCode,
(d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists,
(d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval,
(d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')) AS canCancel
(d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
AND ${rebalanceCancellationGuard}) AS canCancel
`;
function deploymentListQuery(options: ProvisioningListOptions) {
@@ -286,8 +294,9 @@ function deploymentListQuery(options: ProvisioningListOptions) {
OR ${kinds.map(kind => `d.triggered_by LIKE '${kind}:%'`).join(' OR ')})`;
if (options.source === 'campaign' || options.source === 'beta') conditions.push(linked([options.source]));
else if (options.source === 'maintenance') conditions.push(linked(['maintenance', 'watchdog']));
else if (options.source === 'rebalance') conditions.push("d.triggered_by LIKE 'rebalance:%'");
else if (options.source === 'manual' || options.source === 'system') {
conditions.push(`NOT ${linked(['campaign', 'beta', 'maintenance', 'watchdog'])}`);
conditions.push(`NOT ${linked(['campaign', 'beta', 'maintenance', 'watchdog', 'rebalance'])}`);
conditions.push(options.source === 'manual' ? "d.triggered_by='ui'" : "d.triggered_by!='ui'");
}
const search = (options.server ?? '').slice(0, 100).toLowerCase();
@@ -332,7 +341,7 @@ function deploymentListRows(query: ReturnType<typeof deploymentListQuery>, activ
const cache = new Map<string, ProvisioningSource>();
for (const row of rows) {
const sources = byOperation.get(row.id) ?? [];
const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling):(.+)$/);
const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling|rebalance):(.+)$/);
if (fallback && !sources.some(source => source.kind === fallback[1] && source.source_id === fallback[2])) sources.unshift({ kind: fallback[1] as ProvisioningSource['kind'], source_id: fallback[2] });
row.sources = sources.map(source => {
const key = `${source.kind}:${source.source_id}`;