Recover unhealthy containers during create and rebuild
This commit is contained in:
@@ -18,7 +18,7 @@ vi.mock('../../src/lib/server/rebalancing/admission.js', () => ({
|
||||
let queuedKeys = 0;
|
||||
let fixture: Fixture; let hostId: string; let libraryId: string;
|
||||
const image = 'example.test/plutainer:test';
|
||||
const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
|
||||
const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, capabilities: ['instance.unhealthy-recovery'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = ''; queuedKeys = 0;
|
||||
fixture = await newFixture([]);
|
||||
@@ -1503,6 +1503,201 @@ describe('automatic capacity admission and placement', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('unhealthy recovery within native create and rebuild operations', () => {
|
||||
async function launch(action: 'create' | 'rebuild') {
|
||||
const job = await queue(); await advance();
|
||||
if (action === 'create') return job;
|
||||
await acknowledge(job.deploymentId);
|
||||
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const rebuild = queueNativeRebuild(job.name, 'test');
|
||||
if (!rebuild.ok) throw new Error(rebuild.message);
|
||||
await advance();
|
||||
return rebuild;
|
||||
}
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('recovers an unhealthy %s with one restart then a rebuild using the original launch payload', async action => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
Object.assign(config.rcon, { password: '' });
|
||||
const job = await launch(action);
|
||||
const original = (await operationFor(job.deploymentId))!;
|
||||
const currentGeneration = action === 'rebuild' ? 2 : 1;
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconStatusResult).mockClear();
|
||||
remote.state = 'unhealthy'; remote.players = 4;
|
||||
Object.assign(config.rcon, { password: 'changed-after-original-launch' });
|
||||
await acknowledge(job.deploymentId);
|
||||
const restart = (await operationFor(job.deploymentId))!;
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
expect(restart).toMatchObject({ kind: 'instance.restart', state: 'pending', generation: currentGeneration,
|
||||
instanceId: original.instanceId, payload: { unhealthyRecovery: true } });
|
||||
expect(getDeployment(job.deploymentId)!.capacity.pools).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ kind: 'startup', ownsSlot: true }),
|
||||
expect.objectContaining({ kind: 'restart', ownsSlot: true })
|
||||
]));
|
||||
expect(restart.id).not.toBe(original.id);
|
||||
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration);
|
||||
expect(db().prepare('SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId))
|
||||
.toEqual({ recovery_step: 'restart', generation: currentGeneration });
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
|
||||
// Restarting the controller must preserve the current recovery command.
|
||||
closeDb(); await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(restart.id);
|
||||
await acknowledge(job.deploymentId);
|
||||
const rebuild = (await operationFor(job.deploymentId))!;
|
||||
expect(rebuild).toMatchObject({ kind: 'instance.rebuild', state: 'pending', generation: currentGeneration,
|
||||
instanceId: original.instanceId, payload: { unhealthyRecovery: true, nextGeneration: currentGeneration + 1,
|
||||
files: original.payload.files, env: original.payload.env, rconPassword: original.payload.rconPassword,
|
||||
libraryId: original.payload.libraryId, image: original.payload.image } });
|
||||
expect(rebuild.id).not.toBe(restart.id);
|
||||
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration);
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId))
|
||||
.toEqual({ recovery_step: 'rebuild', generation: currentGeneration });
|
||||
remote.state = 'running'; remote.players = 0;
|
||||
await acknowledge(job.deploymentId);
|
||||
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1);
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' });
|
||||
expect(getDeployment(job.deploymentId)!.progress?.recovery?.state).toBe('released');
|
||||
expect(getDeployment(job.deploymentId)!.capacity.pools.every(pool => !pool.ownsSlot)).toBe(true);
|
||||
await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
|
||||
expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1);
|
||||
});
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('finishes %s after a successful recovery restart without another rebuild', async action => {
|
||||
const job = await launch(action);
|
||||
remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
|
||||
const restart = (await operationFor(job.deploymentId))!;
|
||||
expect(restart.kind).toBe('instance.restart');
|
||||
remote.state = 'running'; await acknowledge(job.deploymentId); await advance();
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' });
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(restart.id);
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 2 : 1);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get())
|
||||
.toEqual({ count: action === 'rebuild' ? 1 : 0 });
|
||||
});
|
||||
|
||||
it.each(['restart', 'rebuild'] as const)('retains an unknown recovery %s receipt without replaying or advancing recovery', async step => {
|
||||
const job = await launch('create');
|
||||
remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
|
||||
if (step === 'rebuild') await acknowledge(job.deploymentId);
|
||||
const operation = (await operationFor(job.deploymentId))!;
|
||||
expect(operation.kind).toBe(`instance.${step}`);
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, operation.id, { state: 'unknown', error: 'Reply lost during recovery' });
|
||||
const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb();
|
||||
const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); interruptDeployments();
|
||||
await advance(); await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").get())
|
||||
.toEqual({ count: step === 'restart' ? 2 : 3 });
|
||||
expect((await detail(job.deploymentId)).state).not.toBe('failed');
|
||||
remote.state = 'running'; await acknowledge(job.deploymentId);
|
||||
expect((await detail(job.deploymentId)).state).toBe('done');
|
||||
});
|
||||
|
||||
it.each(['starting', 'restarting', 'unknown'])('waits for %s container readiness without forcing recovery', async state => {
|
||||
const job = await launch('create');
|
||||
const original = (await operationFor(job.deploymentId))!;
|
||||
remote.state = state; await acknowledge(job.deploymentId); await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: `Waiting for container readiness: ${state}` });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT recovery_step FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId))
|
||||
.toEqual({ recovery_step: null });
|
||||
});
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('does not submit recovery if %s is cancelled during its health probe', async action => {
|
||||
const job = await launch(action);
|
||||
const original = (await operationFor(job.deploymentId))!;
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, original.id, { state: 'completed' });
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
let release!: (value: never) => void;
|
||||
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
||||
const pending = advance();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
|
||||
release({ healthy: false, state: 'unhealthy' } as never);
|
||||
await pending; await advance();
|
||||
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('reconciles a delivered recovery rebuild after its %s flow is cancelled', async action => {
|
||||
const job = await launch(action);
|
||||
remote.state = 'unhealthy'; await acknowledge(job.deploymentId); await acknowledge(job.deploymentId);
|
||||
const rebuild = (await operationFor(job.deploymentId))!;
|
||||
expect(rebuild.kind).toBe('instance.rebuild');
|
||||
const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
|
||||
reportHostOperation(hostId, rebuild.id, { state: 'completed' });
|
||||
await advance(); await advance();
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 3 : 2);
|
||||
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
|
||||
});
|
||||
|
||||
it('fails clearly after an unhealthy recovery rebuild instead of looping', async () => {
|
||||
const job = await launch('create');
|
||||
remote.state = 'unhealthy';
|
||||
await acknowledge(job.deploymentId);
|
||||
await acknowledge(job.deploymentId);
|
||||
const rebuild = (await operationFor(job.deploymentId))!;
|
||||
expect(rebuild.kind).toBe('instance.rebuild');
|
||||
await acknowledge(job.deploymentId); await advance();
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: expect.stringMatching(/unhealthy.*restart.*rebuild/i) });
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").get())
|
||||
.toEqual({ count: 3 });
|
||||
});
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('holds unhealthy %s verification until the agent supports recovery', async action => {
|
||||
const job = await launch(action);
|
||||
const original = (await operationFor(job.deploymentId))!;
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory(), capabilities: [] });
|
||||
remote.state = 'unhealthy'; await acknowledge(job.deploymentId); await advance();
|
||||
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
||||
expect(await detail(job.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: expect.stringMatching(/agent/i) });
|
||||
heartbeat(hostId, inventory()); await advance();
|
||||
expect(await operationFor(job.deploymentId)).toMatchObject({ kind: 'instance.restart', payload: { unhealthyRecovery: true } });
|
||||
});
|
||||
|
||||
it.each([false, true])('rebuilds an already unhealthy container without requiring RCON emptiness once supported (supported=%s)', async supported => {
|
||||
const created = await launch('create'); await acknowledge(created.deploymentId);
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
if (!supported) heartbeat(hostId, { ...inventory(), capabilities: [] });
|
||||
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const rebuild = queueNativeRebuild(created.name, 'test');
|
||||
if (!rebuild.ok) throw new Error(rebuild.message);
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconStatusResult).mockClear();
|
||||
remote.state = 'unhealthy'; remote.players = 3;
|
||||
await advance();
|
||||
if (!supported) {
|
||||
expect(await operationFor(rebuild.deploymentId)).toBeNull();
|
||||
expect((await detail(rebuild.deploymentId)).stage).toMatch(/agent/i);
|
||||
heartbeat(hostId, inventory()); await advance();
|
||||
}
|
||||
expect(await operationFor(rebuild.deploymentId)).toMatchObject({ kind: 'instance.rebuild', generation: 1,
|
||||
payload: { unhealthyRecovery: true, nextGeneration: 2 } });
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('native provisioning cancellation and rollback', () => {
|
||||
it('cancels an unhealthy restart and prevents a late health response from reviving it', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
|
||||
Reference in New Issue
Block a user