Recover unhealthy containers during create and rebuild
This commit is contained in:
@@ -44,6 +44,20 @@ it('enforces per-host removal and recovery limits independently of other hosts',
|
||||
expect(concurrencyBlocker('b','restart','other')).toBeNull();
|
||||
});
|
||||
|
||||
it('counts a creation recovery rebuild against rebuild capacity without losing its startup reservation', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
const { concurrencyBlocker, lifecycleHolders } = await import('../../src/lib/server/provisioning/concurrency.js');
|
||||
db().prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('i','t5_test','host','lib','img',1,'2026-01-01')").run();
|
||||
db().prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','create','running','recovery','2026-01-01','test')").run();
|
||||
db().prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,recovery_step) VALUES('a','i',1,'rebuild')").run();
|
||||
saveFleetSettings({ ...fleetSettings(), rebuildConcurrency: 1, createConcurrencyPerHost: 1 });
|
||||
expect(lifecycleHolders()).toEqual([expect.objectContaining({ id: 'a', action: 'rebuild' })]);
|
||||
expect(concurrencyBlocker('b', 'rebuild', 'other')).toContain('rebuild');
|
||||
expect(concurrencyBlocker('b', 'create', 'host')).toContain('startup');
|
||||
expect(concurrencyBlocker('a', 'rebuild', 'host')).toBeNull();
|
||||
});
|
||||
|
||||
it('retains every capacity owner across failure, cancellation and migration without counting completed history', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { createHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
@@ -99,7 +113,7 @@ it('seeks capacity owners through active indexes instead of scanning retained jo
|
||||
let sql: string;
|
||||
try {
|
||||
expect(lifecycleHolders()).toEqual([]);
|
||||
sql = prepare.mock.calls.find(([sql])=>sql.startsWith('SELECT d.id,d.action'))![0];
|
||||
sql = prepare.mock.calls.find(([sql])=>sql.startsWith('SELECT d.id,'))![0];
|
||||
} finally { prepare.mockRestore(); }
|
||||
const plan = h.prepare(`EXPLAIN QUERY PLAN ${sql}`).all('[]') as {detail:string}[];
|
||||
const details = plan.map(row=>row.detail).join('\n');
|
||||
|
||||
@@ -18,7 +18,7 @@ vi.mock('../../src/lib/server/rebalancing/admission.js', () => ({
|
||||
let queuedKeys = 0;
|
||||
let fixture: Fixture; let hostId: string; let libraryId: string;
|
||||
const image = 'example.test/plutainer:test';
|
||||
const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
|
||||
const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, capabilities: ['instance.unhealthy-recovery'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = ''; queuedKeys = 0;
|
||||
fixture = await newFixture([]);
|
||||
@@ -1503,6 +1503,201 @@ describe('automatic capacity admission and placement', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('unhealthy recovery within native create and rebuild operations', () => {
|
||||
async function launch(action: 'create' | 'rebuild') {
|
||||
const job = await queue(); await advance();
|
||||
if (action === 'create') return job;
|
||||
await acknowledge(job.deploymentId);
|
||||
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const rebuild = queueNativeRebuild(job.name, 'test');
|
||||
if (!rebuild.ok) throw new Error(rebuild.message);
|
||||
await advance();
|
||||
return rebuild;
|
||||
}
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('recovers an unhealthy %s with one restart then a rebuild using the original launch payload', async action => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
Object.assign(config.rcon, { password: '' });
|
||||
const job = await launch(action);
|
||||
const original = (await operationFor(job.deploymentId))!;
|
||||
const currentGeneration = action === 'rebuild' ? 2 : 1;
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconStatusResult).mockClear();
|
||||
remote.state = 'unhealthy'; remote.players = 4;
|
||||
Object.assign(config.rcon, { password: 'changed-after-original-launch' });
|
||||
await acknowledge(job.deploymentId);
|
||||
const restart = (await operationFor(job.deploymentId))!;
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
expect(restart).toMatchObject({ kind: 'instance.restart', state: 'pending', generation: currentGeneration,
|
||||
instanceId: original.instanceId, payload: { unhealthyRecovery: true } });
|
||||
expect(getDeployment(job.deploymentId)!.capacity.pools).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ kind: 'startup', ownsSlot: true }),
|
||||
expect.objectContaining({ kind: 'restart', ownsSlot: true })
|
||||
]));
|
||||
expect(restart.id).not.toBe(original.id);
|
||||
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration);
|
||||
expect(db().prepare('SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId))
|
||||
.toEqual({ recovery_step: 'restart', generation: currentGeneration });
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
|
||||
// Restarting the controller must preserve the current recovery command.
|
||||
closeDb(); await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(restart.id);
|
||||
await acknowledge(job.deploymentId);
|
||||
const rebuild = (await operationFor(job.deploymentId))!;
|
||||
expect(rebuild).toMatchObject({ kind: 'instance.rebuild', state: 'pending', generation: currentGeneration,
|
||||
instanceId: original.instanceId, payload: { unhealthyRecovery: true, nextGeneration: currentGeneration + 1,
|
||||
files: original.payload.files, env: original.payload.env, rconPassword: original.payload.rconPassword,
|
||||
libraryId: original.payload.libraryId, image: original.payload.image } });
|
||||
expect(rebuild.id).not.toBe(restart.id);
|
||||
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration);
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId))
|
||||
.toEqual({ recovery_step: 'rebuild', generation: currentGeneration });
|
||||
remote.state = 'running'; remote.players = 0;
|
||||
await acknowledge(job.deploymentId);
|
||||
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1);
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' });
|
||||
expect(getDeployment(job.deploymentId)!.progress?.recovery?.state).toBe('released');
|
||||
expect(getDeployment(job.deploymentId)!.capacity.pools.every(pool => !pool.ownsSlot)).toBe(true);
|
||||
await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
|
||||
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1);
|
||||
});
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('finishes %s after a successful recovery restart without another rebuild', async action => {
|
||||
const job = await launch(action);
|
||||
remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
|
||||
const restart = (await operationFor(job.deploymentId))!;
|
||||
expect(restart.kind).toBe('instance.restart');
|
||||
remote.state = 'running'; await acknowledge(job.deploymentId); await advance();
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' });
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(restart.id);
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 2 : 1);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get())
|
||||
.toEqual({ count: action === 'rebuild' ? 1 : 0 });
|
||||
});
|
||||
|
||||
it.each(['restart', 'rebuild'] as const)('retains an unknown recovery %s receipt without replaying or advancing recovery', async step => {
|
||||
const job = await launch('create');
|
||||
remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
|
||||
if (step === 'rebuild') await acknowledge(job.deploymentId);
|
||||
const operation = (await operationFor(job.deploymentId))!;
|
||||
expect(operation.kind).toBe(`instance.${step}`);
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, operation.id, { state: 'unknown', error: 'Reply lost during recovery' });
|
||||
const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb();
|
||||
const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); interruptDeployments();
|
||||
await advance(); await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").get())
|
||||
.toEqual({ count: step === 'restart' ? 2 : 3 });
|
||||
expect((await detail(job.deploymentId)).state).not.toBe('failed');
|
||||
remote.state = 'running'; await acknowledge(job.deploymentId);
|
||||
expect((await detail(job.deploymentId)).state).toBe('done');
|
||||
});
|
||||
|
||||
it.each(['starting', 'restarting', 'unknown'])('waits for %s container readiness without forcing recovery', async state => {
|
||||
const job = await launch('create');
|
||||
const original = (await operationFor(job.deploymentId))!;
|
||||
remote.state = state; await acknowledge(job.deploymentId); await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: `Waiting for container readiness: ${state}` });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT recovery_step FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId))
|
||||
.toEqual({ recovery_step: null });
|
||||
});
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('does not submit recovery if %s is cancelled during its health probe', async action => {
|
||||
const job = await launch(action);
|
||||
const original = (await operationFor(job.deploymentId))!;
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, original.id, { state: 'completed' });
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
let release!: (value: never) => void;
|
||||
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
||||
const pending = advance();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
|
||||
release({ healthy: false, state: 'unhealthy' } as never);
|
||||
await pending; await advance();
|
||||
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('reconciles a delivered recovery rebuild after its %s flow is cancelled', async action => {
|
||||
const job = await launch(action);
|
||||
remote.state = 'unhealthy'; await acknowledge(job.deploymentId); await acknowledge(job.deploymentId);
|
||||
const rebuild = (await operationFor(job.deploymentId))!;
|
||||
expect(rebuild.kind).toBe('instance.rebuild');
|
||||
const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
|
||||
reportHostOperation(hostId, rebuild.id, { state: 'completed' });
|
||||
await advance(); await advance();
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 3 : 2);
|
||||
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
|
||||
});
|
||||
|
||||
it('fails clearly after an unhealthy recovery rebuild instead of looping', async () => {
|
||||
const job = await launch('create');
|
||||
remote.state = 'unhealthy';
|
||||
await acknowledge(job.deploymentId);
|
||||
await acknowledge(job.deploymentId);
|
||||
const rebuild = (await operationFor(job.deploymentId))!;
|
||||
expect(rebuild.kind).toBe('instance.rebuild');
|
||||
await acknowledge(job.deploymentId); await advance();
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: expect.stringMatching(/unhealthy.*restart.*rebuild/i) });
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").get())
|
||||
.toEqual({ count: 3 });
|
||||
});
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('holds unhealthy %s verification until the agent supports recovery', async action => {
|
||||
const job = await launch(action);
|
||||
const original = (await operationFor(job.deploymentId))!;
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory(), capabilities: [] });
|
||||
remote.state = 'unhealthy'; await acknowledge(job.deploymentId); await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: expect.stringMatching(/agent/i) });
|
||||
heartbeat(hostId, inventory()); await advance();
|
||||
expect(await operationFor(job.deploymentId)).toMatchObject({ kind: 'instance.restart', payload: { unhealthyRecovery: true } });
|
||||
});
|
||||
|
||||
it.each([false, true])('rebuilds an already unhealthy container without requiring RCON emptiness once supported (supported=%s)', async supported => {
|
||||
const created = await launch('create'); await acknowledge(created.deploymentId);
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
if (!supported) heartbeat(hostId, { ...inventory(), capabilities: [] });
|
||||
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const rebuild = queueNativeRebuild(created.name, 'test');
|
||||
if (!rebuild.ok) throw new Error(rebuild.message);
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconStatusResult).mockClear();
|
||||
remote.state = 'unhealthy'; remote.players = 3;
|
||||
await advance();
|
||||
if (!supported) {
|
||||
expect(await operationFor(rebuild.deploymentId)).toBeNull();
|
||||
expect((await detail(rebuild.deploymentId)).stage).toMatch(/agent/i);
|
||||
heartbeat(hostId, inventory()); await advance();
|
||||
}
|
||||
expect(await operationFor(rebuild.deploymentId)).toMatchObject({ kind: 'instance.rebuild', generation: 1,
|
||||
payload: { unhealthyRecovery: true, nextGeneration: 2 } });
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('native provisioning cancellation and rollback', () => {
|
||||
it('cancels an unhealthy restart and prevents a late health response from reviving it', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { prepareNativeHost } from '../helpers/native-host.js';
|
||||
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const server = 't5_zombie_theater_1';
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture([]);
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); });
|
||||
|
||||
async function recovering(step: 'restart' | 'rebuild') {
|
||||
const { host } = await prepareNativeHost([server]);
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
const { processNativeProvisioning, reconcileNativeProvisioningReceipt } = await import('../../src/lib/server/provisioning/native.js');
|
||||
hosts.heartbeat(host.id, { ...hosts.getHost(host.id)!.inventory, capabilities: ['instance.cancel-create', 'instance.unhealthy-recovery'] });
|
||||
db().prepare('INSERT INTO servers(name) VALUES(?)').run(server);
|
||||
const instance = nativeInstance(server)!;
|
||||
hosts.reserveHostPort(host.id, instance.id, { port: instance.port, protocol: 'udp' });
|
||||
hosts.reserveHostPort(host.id, instance.id, { port: instance.port, protocol: 'tcp' });
|
||||
const deploymentId = beginDeployment(server, 'create');
|
||||
const original = hosts.queueHostOperation(host.id, 'instance.create', {}, { instanceId: instance.id, generation: 1 });
|
||||
hosts.reportHostOperation(host.id, original.id, { state: 'completed', result: { state: 'unhealthy', healthy: false } });
|
||||
const recovery = hosts.queueHostOperation(host.id, `instance.${step}`, { unhealthyRecovery: true, ...(step === 'rebuild' ? { nextGeneration: 2 } : {}) },
|
||||
{ instanceId: instance.id, generation: 1 });
|
||||
db().prepare('UPDATE native_provisioning SET operation_id=?,recovery_step=? WHERE deployment_id=?').run(recovery.id, step, deploymentId);
|
||||
db().prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?").run(deploymentId);
|
||||
const commandFor = (id: string) => {
|
||||
const row = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(id) as { operation_id: string };
|
||||
return hosts.getHostOperation(row.operation_id)!;
|
||||
};
|
||||
return { host, instance, deploymentId, original, recovery, ...hosts, db, nativeInstance, cancelCreation, canCancelCreation,
|
||||
getDeployment, reconcileNativeProvisioningReceipt, commandFor, advance: () => processNativeProvisioning(() => null) };
|
||||
}
|
||||
|
||||
describe('cancelling creation during unhealthy recovery', () => {
|
||||
it('keeps the original creation identity and suppresses recovery restart replay', async () => {
|
||||
const api = await recovering('restart');
|
||||
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toContain(api.recovery.id);
|
||||
const cleanup = api.cancelCreation(api.deploymentId)!;
|
||||
const removal = api.commandFor(cleanup);
|
||||
expect(removal).toMatchObject({ kind: 'instance.cancel-create', generation: 1, payload: { createOperationId: api.original.id } });
|
||||
expect(removal.payload).toEqual({ createOperationId: api.original.id });
|
||||
expect(api.pendingCreationCancellations(api.host.id)).toEqual([api.original.id]);
|
||||
expect(api.pendingHostOperations(api.host.id).filter(operation => operation.kind.startsWith('instance.')).map(operation => operation.id)).toEqual([removal.id]);
|
||||
api.reportHostOperation(api.host.id, removal.id, { state: 'completed', result: { exists: false } });
|
||||
await api.advance();
|
||||
expect(api.getDeployment(cleanup)?.deployment.state).toBe('done');
|
||||
expect(api.getHostOperation(api.recovery.id)?.state).toBe('failed');
|
||||
expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['pending', 'running', 'unknown', 'completed'] as const)('cleans up both rebuild generations when recovery is %s', async state => {
|
||||
const api = await recovering('rebuild');
|
||||
if (state !== 'pending') {
|
||||
api.pendingHostOperations(api.host.id);
|
||||
api.reportHostOperation(api.host.id, api.recovery.id, { state, result: { state: 'unhealthy', healthy: false } });
|
||||
}
|
||||
if (state === 'completed') api.reconcileNativeProvisioningReceipt(api.deploymentId);
|
||||
const registeredGeneration = state === 'completed' ? 2 : 1;
|
||||
expect(api.nativeInstance(server)?.generation).toBe(registeredGeneration);
|
||||
expect(api.canCancelCreation(api.deploymentId)).toBe(true);
|
||||
const cleanup = api.cancelCreation(api.deploymentId)!;
|
||||
const removal = api.commandFor(cleanup);
|
||||
expect(removal).toMatchObject({ kind: 'instance.cancel-create', generation: 1,
|
||||
payload: { createOperationId: api.original.id, recoveryRebuildOperationId: api.recovery.id, recoveryGeneration: 2 } });
|
||||
expect(api.pendingCreationCancellations(api.host.id)).toEqual([api.original.id, api.recovery.id]);
|
||||
expect(api.db().prepare('SELECT generation FROM native_provisioning WHERE deployment_id=?').get(cleanup)).toEqual({ generation: registeredGeneration });
|
||||
expect(api.pendingHostOperations(api.host.id).filter(operation => operation.kind.startsWith('instance.')).map(operation => operation.id)).toEqual([removal.id]);
|
||||
// A delivered rebuild may report success after cancellation, while the
|
||||
// cleanup command already owns deletion of both possible containers.
|
||||
api.reportHostOperation(api.host.id, api.recovery.id, { state: 'completed', result: { state: 'unhealthy', healthy: false } });
|
||||
api.reconcileNativeProvisioningReceipt(api.deploymentId);
|
||||
expect(api.nativeInstance(server)?.generation).toBe(registeredGeneration);
|
||||
expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
|
||||
api.reportHostOperation(api.host.id, removal.id, { state: 'completed', result: { exists: false } });
|
||||
await api.advance();
|
||||
expect(api.getDeployment(cleanup)?.deployment.state).toBe('done');
|
||||
expect(api.getDeployment(api.deploymentId)?.deployment).toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) });
|
||||
expect(api.nativeInstance(server)).toBeNull();
|
||||
expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
||||
expect(api.db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toBeUndefined();
|
||||
expect(api.pendingCreationCancellations(api.host.id)).toEqual([]);
|
||||
expect(api.cancelCreation(api.deploymentId)).toBe(cleanup);
|
||||
});
|
||||
|
||||
it('does not adopt a changed generation without a completed recovery receipt', async () => {
|
||||
const api = await recovering('rebuild');
|
||||
api.db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(api.instance.id);
|
||||
expect(api.canCancelCreation(api.deploymentId)).toBe(false);
|
||||
expect(api.cancelCreation(api.deploymentId)).toBeNull();
|
||||
api.reportHostOperation(api.host.id, api.recovery.id, { state: 'completed' });
|
||||
api.db().prepare('UPDATE native_instances SET generation=3 WHERE id=?').run(api.instance.id);
|
||||
expect(api.cancelCreation(api.deploymentId)).toBeNull();
|
||||
});
|
||||
|
||||
it('holds extended cleanup until the host supports recovery cancellation', async () => {
|
||||
const api = await recovering('rebuild');
|
||||
const cleanup = api.cancelCreation(api.deploymentId)!;
|
||||
api.heartbeat(api.host.id, { ...api.getHost(api.host.id)!.inventory, capabilities: ['instance.cancel-create'] });
|
||||
expect(api.pendingHostOperations(api.host.id)).toEqual([]);
|
||||
await api.advance();
|
||||
expect(api.getDeployment(cleanup)?.deployment.stage).toContain('Update the host agent');
|
||||
expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
|
||||
api.heartbeat(api.host.id, { ...api.getHost(api.host.id)!.inventory, capabilities: ['instance.cancel-create', 'instance.unhealthy-recovery'] });
|
||||
expect(api.pendingHostOperations(api.host.id).filter(operation => operation.kind.startsWith('instance.')).map(operation => operation.id)).toEqual([api.commandFor(cleanup).id]);
|
||||
});
|
||||
});
|
||||
@@ -230,3 +230,83 @@ it('protects cancelled workflows and linked file history through ambiguous and u
|
||||
expect(a.pruneHistory('provisioning')).toBe(1);
|
||||
expect(a.db.pragma('foreign_key_check')).toEqual([]);
|
||||
});
|
||||
|
||||
async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
|
||||
const a = await api();
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.createHost({ name: 'Recovery host' });
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1, dockerRunning: true,
|
||||
directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [config.defaultRuntimeImage],
|
||||
libraries: [{ id: 'lib', state: 'ready' }] });
|
||||
const generation = action === 'create' ? 1 : 2;
|
||||
a.db.prepare("INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,?,'lib','image',1,?)")
|
||||
.run(host.id, generation, old);
|
||||
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('recovering','s',?,'awaiting-health','Recovery',?,'test')")
|
||||
.run(action === 'create' ? 'create' : 'rebuild', old);
|
||||
const sha256 = putFileBlob(Buffer.from('original launch files'), a.db);
|
||||
const launch = hosts.queueHostOperation(host.id, action === 'rebuild' ? 'instance.rebuild' : 'instance.create', {
|
||||
name: 's', files: [{ path: 'configs/server.cfg', sha256, contentSha256: sha256 }]
|
||||
}, { instanceId: 'instance', generation: action === 'rebuild' ? 1 : generation,
|
||||
idempotencyKey: action === 'migration' ? 'recovering:migration-create:0' : 'recovering' });
|
||||
hosts.reportHostOperation(host.id, launch.id, { state: 'completed' });
|
||||
const restart = hosts.queueHostOperation(host.id, 'instance.restart', { unhealthyRecovery: true }, { instanceId: 'instance', generation });
|
||||
a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,recovery_step) VALUES('recovering','instance',?,?,'restart')")
|
||||
.run(generation, restart.id);
|
||||
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
|
||||
a.saveFleetSettings({ ...a.fleetSettings(), actionKeepDays: 1, payloadKeepDays: 1, historyKeepCount: 0 });
|
||||
return { ...a, hosts, host, generation, launch, restart, sha256 };
|
||||
}
|
||||
|
||||
it.each(['create', 'rebuild', 'migration'] as const)('keeps original %s launch files and ownership through recovery and releases them after success', async action => {
|
||||
const a = await recoveryLaunch(action);
|
||||
expect(a.pruneHistory('operation-payload')).toBe(0);
|
||||
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([
|
||||
{ path: 'configs/server.cfg', sha256: a.sha256, contentBase64: Buffer.from('original launch files').toString('base64') }
|
||||
]);
|
||||
a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' });
|
||||
const rebuild = a.hosts.queueHostOperation(a.host.id, 'instance.rebuild', { unhealthyRecovery: true, nextGeneration: a.generation + 1 },
|
||||
{ instanceId: 'instance', generation: a.generation });
|
||||
a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' });
|
||||
a.db.prepare("UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'")
|
||||
.run(rebuild.id, old);
|
||||
a.db.prepare("UPDATE native_instances SET generation=generation+1 WHERE id='instance'").run();
|
||||
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
|
||||
a.pruneHistory('operation-payload');
|
||||
a.pruneHistory('operation');
|
||||
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
|
||||
.toEqual({ history_expired_at: null });
|
||||
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([
|
||||
{ path: 'configs/server.cfg', sha256: a.sha256, contentBase64: Buffer.from('original launch files').toString('base64') }
|
||||
]);
|
||||
a.db.prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id='recovering'").run(old);
|
||||
a.pruneHistory('operation-payload');
|
||||
expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([{ path: 'configs/server.cfg', sha256: a.sha256 }]);
|
||||
a.pruneHistory('operation');
|
||||
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
|
||||
.toEqual({ history_expired_at: expect.any(String) });
|
||||
});
|
||||
|
||||
it.each(['cancelled', 'failed'])('keeps original ownership for a %s recovered creation until a later lifecycle operation supersedes cleanup', async state => {
|
||||
const a = await recoveryLaunch('create');
|
||||
a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' });
|
||||
const rebuild = a.hosts.queueHostOperation(a.host.id, 'instance.rebuild', { unhealthyRecovery: true, nextGeneration: 2 },
|
||||
{ instanceId: 'instance', generation: 1 });
|
||||
a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' });
|
||||
a.db.prepare("UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'")
|
||||
.run(rebuild.id, old);
|
||||
a.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run();
|
||||
a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old);
|
||||
a.db.prepare("UPDATE server_deployments SET state=?,finished_at=? WHERE id='recovering'").run(state, old);
|
||||
const { canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
expect(canCancelCreation('recovering')).toBe(true);
|
||||
a.pruneHistory('operation');
|
||||
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
|
||||
.toEqual({ history_expired_at: null });
|
||||
a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('replacement','s','restart','done','Done',?,?,'test')")
|
||||
.run(old, old);
|
||||
expect(canCancelCreation('recovering')).toBe(false);
|
||||
a.pruneHistory('operation');
|
||||
expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id))
|
||||
.toEqual({ history_expired_at: expect.any(String) });
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user