josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Reconcile rebuilds interrupted before replacement creation

Browse Source
This commit is contained in:
josh
2026-10-08 14:24:52 -04:00
parent e01b143a09
commit c4e2ff4384
11 changed files with 571 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/cancellation.ts
+11 -5
View File
@@ -60,14 +60,20 @@ export function cancelProvisioningJob(id: string, reason = 'Cancelled by operato
}
/** A cancelled controller job must not let a replacement overtake a delivered mutation. */
export function unsettledCancelledCommand(server: string, replacementId: string): string | null {
const row = db().prepare(`SELECT d.id FROM server_deployments d
export function unsettledCancelledCommand(server: string, replacementId: string): {
deploymentId: string; operationId: string; state: string; error: string | null;
} | null {
const row = db().prepare(`SELECT d.id AS deploymentId,o.id AS operationId,o.state,o.error FROM server_deployments d
JOIN native_provisioning n ON n.deployment_id=d.id
LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
JOIN ec_host_operations o ON o.id=n.operation_id OR o.id=m.remove_operation_id
WHERE d.server=? AND d.server_deleted_at IS NULL AND d.state='cancelled'
AND (o.state NOT IN ('completed','failed')
OR (o.state='completed' AND o.id=n.operation_id AND n.files_json IS NOT NULL AND n.applied_at IS NULL))
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id AND c.cleanup_id=?) LIMIT 1`).get(server, replacementId) as { id: string } | undefined;
return row?.id ?? null;
OR (o.state='completed' AND o.id=n.operation_id AND n.files_json IS NOT NULL AND n.applied_at IS NULL)
OR (o.state='failed' AND o.id=n.operation_id AND o.kind='instance.rebuild'
AND json_type(o.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL))
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id AND c.cleanup_id=?)
ORDER BY d.rowid,o.rowid LIMIT 1`).get(server, replacementId) as
{ deploymentId: string; operationId: string; state: string; error: string | null } | undefined;
return row ?? null;
}
src/lib/server/provisioning/native.ts
+56 -4
View File
@@ -13,7 +13,7 @@ import { ensureHostSchema } from '../hosts/schema.js';
import { unsettledCancelledCommand } from './cancellation.js';
import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
import { getHost, listHosts, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { getHost, listHosts, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort, type HostOperationStatus } from '../hosts/index.js';
import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeSummaryForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
@@ -277,6 +277,48 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
})();
}
/** Consume a conclusive partial failure without treating removal as a healthy rebuild. */
function applyIncompleteRebuildReceipt(job: Job, operation: HostOperationStatus): void {
const result = operation.result as { reconciled?: unknown; rebuildFailure?: {
phase?: unknown; generation?: unknown; nextGeneration?: unknown;
oldContainerMissing?: unknown; replacementContainerMissing?: unknown; replacementFilesRemoved?: unknown;
} | null } | null;
if (!result || !Object.prototype.hasOwnProperty.call(result, 'rebuildFailure')) return;
const failure = result.rebuildFailure;
if (operation.state !== 'failed' || operation.kind !== 'instance.rebuild' || result.reconciled !== true
|| !failure || failure.phase !== 'disrupting' || failure.generation !== job.generation
|| failure.nextGeneration !== job.generation + 1 || failure.oldContainerMissing !== true
|| failure.replacementContainerMissing !== true || failure.replacementFilesRemoved !== true) {
throw new Error('The interrupted rebuild receipt does not confirm safe replacement cleanup. Inspect the original host command.');
}
db().transaction(() => {
const binding = db().prepare('SELECT operation_id,applied_at FROM native_provisioning WHERE deployment_id=?')
.get(job.id) as { operation_id: string | null; applied_at: string | null } | undefined;
if (binding?.operation_id !== operation.id) throw new Error('The interrupted rebuild command no longer belongs to this operation.');
if (binding.applied_at) return;
const requested = db().prepare(`SELECT COALESCE(json_extract(COALESCE(p.payload,o.payload),'$.nextGeneration'),
json_extract(COALESCE(p.payload,o.payload),'$.newGeneration')) AS nextGeneration FROM ec_host_operations o
LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(operation.id) as
{ nextGeneration: number | null } | undefined;
if (requested?.nextGeneration !== failure.nextGeneration) {
throw new Error('The interrupted rebuild receipt does not match its requested replacement generation.');
}
const instance = nativeInstance(job.server);
if (!instance || instance.id !== job.instance_id || instance.generation !== job.generation
|| operation.instanceId !== instance.id || operation.hostId !== instance.host_id || operation.generation !== instance.generation) {
throw new Error('The interrupted rebuild belongs to a different instance or generation. Inspect its host command.');
}
db().prepare("UPDATE servers SET container_state='missing',container_checked_at=?,last_players=NULL,last_roster=NULL WHERE name=?")
.run(operation.updatedAt, job.server);
db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
recordProgress(job.id, { checks: [{ key: 'container', label: 'Container state after interrupted rebuild', source: 'agent',
result: 'fail', actual: 'missing', checkedAt: operation.updatedAt,
message: 'The old container is absent and replacement creation never started. Partial replacement files were removed.' }],
event: { kind: 'reconciled', message: 'Interrupted rebuild settled: old container removed; replacement creation failed.',
details: { operationId: operation.id, generation: job.generation, nextGeneration: failure.nextGeneration } } });
})();
}
/** Retain confirmed file snapshots and instance identity even after observation is cancelled. */
export function reconcileNativeProvisioningReceipt(id: string): void {
const job = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.operation_id,n.applied_at,n.recovery_step
@@ -303,6 +345,7 @@ export function reconcileNativeProvisioningReceipt(id: string): void {
if (!job.operation_id) return;
const operation = getHostOperationStatus(job.operation_id);
if (operation?.state === 'failed') {
applyIncompleteRebuildReceipt(job, operation);
for (const file of storedFiles(job)) {
db().prepare("UPDATE staged_file_changes SET state=? WHERE id=? AND state='applying'")
.run(job.state === 'cancelled' ? 'cancelled' : 'pending', file.id);
@@ -434,6 +477,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
LEFT JOIN ec_host_operations removal ON removal.id=m.remove_operation_id
WHERE d.state='cancelled' AND d.server_deleted_at IS NULL AND (
(o.state='completed' AND (n.applied_at IS NULL OR d.action='destroy'))
OR (o.state='failed' AND o.kind='instance.rebuild' AND json_type(o.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL)
OR (o.state='failed' AND EXISTS (SELECT 1 FROM staged_file_changes f JOIN provisioning_sources s
ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
WHERE s.operation_id=d.id AND f.state='applying'))
@@ -542,7 +586,14 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null, el
async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (!stillActive(job)) return;
const unsettled = unsettledCancelledCommand(job.server, job.id);
if (unsettled) { hold(job, 'Waiting for the already-delivered command from cancelled operation ' + unsettled + ' to settle.'); return; }
if (unsettled) {
const pendingReceipt = unsettled.state === 'completed' || unsettled.state === 'failed';
const message = `Waiting for the host command from cancelled operation ${unsettled.deploymentId} ${pendingReceipt ? 'to be reconciled' : 'to settle'}. Agent state: ${unsettled.state}.`
+ (unsettled.error ? ` Last error: ${unsettled.error.slice(0, 500)}` : '');
deploymentStage(job.id, message, { blocker: { code: 'cancelled-command', message, automatic: true,
href: `/provisioning/${encodeURIComponent(unsettled.deploymentId)}`, label: 'View cancelled operation' } });
return;
}
let instance = nativeInstance(job.server);
if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) {
// A resumed bound job owns its original identity. An unplaced intent
@@ -588,6 +639,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
}
if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
if (operation.state === 'failed') {
applyIncompleteRebuildReceipt(job, operation);
if (job.action === 'destroy' && scalingIntent(job.id) && (operation.result as { autoscalingInvalidated?: boolean } | null)?.autoscalingInvalidated === true) {
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state='cancelled',stage='Automatic removal withdrawn; new empty interval required',finished_at=?,error=NULL WHERE id=?")
@@ -607,8 +659,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, `${operation.error ?? 'Destination creation failed before mutation.'} Retrying after fresh capacity checks.`);
return;
}
// The agent reports failed only when no mutation began. Ambiguous
// post-mutation failures remain unknown and retain their capacity.
// Failed commands either never mutated or have a conclusively
// reconciled partial rebuild above. Ambiguous outcomes retain capacity.
db().transaction(() => {
db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id);
const files = storedFiles(job);