josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Offer rebuild retry for failed creation commands

Browse Source
This commit is contained in:
josh
2026-10-08 14:05:38 -04:00
parent 511f090552
commit e01b143a09
12 changed files with 298 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/history.ts
+35 -7
View File
@@ -14,6 +14,7 @@ import { canCancelCreation } from './cancel-creation.js';
import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js';
import { committedRebalanceStartup } from './concurrency.js';
import { nativeDeployment } from '../backends/native.js';
import { ensureHostSchema } from '../hosts/schema.js';
import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
import { serverTitle } from '../../ui/server-labels.js';
import { provisioningQueueStatus } from '../../ui/provisioning-queue.js';
@@ -149,7 +150,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
canResumeRebuild: canResumeRebuild(id), canRetryWithRebuild: canRetryWithRebuild(id),
canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
warningMessages: warningMessages(deployment.warnings), generatedAt };
})();
@@ -197,17 +199,43 @@ export function resumeCleanup(id: string): boolean {
}
export function canResumeVerification(id: string): boolean {
ensureHostSchema(db());
return !!db().prepare(`SELECT d.id FROM server_deployments d
JOIN server_creation_jobs j ON j.deployment_id = d.id
JOIN servers s ON s.name = j.server AND s.game_port = j.port
WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND (EXISTS (
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL) OR EXISTS (
SELECT 1 FROM ec_host_migrations m JOIN native_provisioning n ON n.deployment_id=m.deployment_id
WHERE m.deployment_id=d.id AND m.removed_at IS NOT NULL AND n.operation_id IS NOT NULL)))) AND d.state IN ('failed', 'interrupted')
JOIN servers s ON s.name = j.server AND s.game_port = j.port AND s.destroyed_at IS NULL
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
LEFT JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
LEFT JOIN ec_host_operations o ON o.id=n.operation_id
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('create','rebuild')
AND d.state IN ('failed', 'interrupted')
AND ((n.deployment_id IS NOT NULL AND i.port=j.port AND o.state='completed'
AND o.instance_id=i.id AND o.host_id=i.host_id AND o.generation=n.generation
AND ((o.kind IN ('instance.create','instance.restart') AND i.generation=n.generation)
OR (o.kind='instance.rebuild' AND i.generation IN (n.generation,n.generation+1))))
OR (n.deployment_id IS NULL AND (d.action='create' OR EXISTS (
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id=d.id AND r.removed_at IS NOT NULL))))
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id);
}
/** A conclusively failed initial create can enter the normal rebuild flow. */
export function canRetryWithRebuild(id: string): boolean {
ensureHostSchema(db());
return !!db().prepare(`SELECT d.id FROM server_deployments d
JOIN server_creation_jobs j ON j.deployment_id=d.id
JOIN servers s ON s.name=d.server AND s.game_port=j.port AND s.destroyed_at IS NULL
JOIN native_provisioning n ON n.deployment_id=d.id AND n.recovery_step IS NULL
JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
AND i.generation=n.generation AND i.port=j.port
JOIN ec_host_operations o ON o.id=n.operation_id AND o.instance_id=i.id
AND o.host_id=i.host_id AND o.generation=n.generation
WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action='create'
AND d.state IN ('failed','interrupted') AND o.kind='instance.create' AND o.state='failed'
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id)
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)`)
.get(id);
}
export function canResumeRebuild(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
@@ -226,7 +254,7 @@ export function resumeRebuild(id: string): boolean {
})();
}
/** Resume observation only. Never repeat a container creation or restart. */
/** Resume readiness after a completed command; unhealthy recovery still applies. */
export function resumeVerification(id: string): boolean {
return db().transaction(() => {
if (!canResumeVerification(id)) return false;