Cancel active provisioning jobs when rolling back betas
This commit is contained in:
@@ -175,7 +175,8 @@ describe('durable beta test plans', () => {
|
||||
const { getBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(getBeta(id)?.state).toBe('rolled-back');
|
||||
expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('main() {}');
|
||||
// Rollback now replaces the unhealthy restart in the same worker pass.
|
||||
expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('original');
|
||||
expect(existsSync(scriptPath(names[1]))).toBe(false);
|
||||
expect(db().prepare("SELECT server, restart_state FROM deployment_servers").all()).toEqual([{ server: names[0], restart_state: 'pending' }]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
|
||||
@@ -20,7 +20,7 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'localhost', port: 27016 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const serverName = 't5_zombie_cosmodrome_1';
|
||||
@@ -68,9 +68,9 @@ describe('queued removal cancellation', () => {
|
||||
resolveCount(0);
|
||||
await work;
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', server_exists: 1, stage: 'Removal cancelled' });
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', server_exists: 1, stage: 'Cancelled by operator' });
|
||||
expect(getDeployment(queued.deploymentId!)?.canCancelRemoval).toBe(false);
|
||||
expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(409);
|
||||
expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(200);
|
||||
});
|
||||
|
||||
it('does not overwrite cancellation when the in-flight player check rejects', async () => {
|
||||
@@ -760,3 +760,44 @@ describe('persistent server keys', () => {
|
||||
.toMatchObject({ ok: true, name: serverName });
|
||||
});
|
||||
});
|
||||
|
||||
describe('cancelling submitted creation and destruction', () => {
|
||||
it('retains a remotely created identity after cancellation without starting its container', async () => {
|
||||
const { queueCreateServer, processCreations, nextAvailablePort, nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const queued = queueCreateServer(input); if (!queued.ok) throw new Error(queued.message);
|
||||
let release!: (value: {ok:true}) => void;
|
||||
vi.mocked(komodoCreateDeployment).mockImplementationOnce(() => new Promise(resolve => {release=resolve;}));
|
||||
const work = processCreations();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(cancelProvisioningJob(queued.deploymentId!)).toBe(true);
|
||||
expect(nextAvailablePort()).not.toBe(queued.port);
|
||||
expect(nextLobbyNumber('t5_zombie_cosmodrome')).toBe(2);
|
||||
release({ok:true}); await work;
|
||||
expect(getServer(queued.name)).toBeTruthy();
|
||||
expect(komodoDeploy).not.toHaveBeenCalled();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment.state).toBe('cancelled');
|
||||
});
|
||||
|
||||
it('does not delete deployment metadata or files after cancelling container teardown', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await createServer(input);
|
||||
let release!: (value: {ok:true}) => void;
|
||||
vi.mocked(komodoDestroyDeployment).mockImplementationOnce(() => new Promise(resolve => {release=resolve;}));
|
||||
const queued=destroyServer(serverName); const work=processDestructions();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(cancelProvisioningJob(queued.deploymentId!)).toBe(true);
|
||||
release({ok:true}); await work;
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
expect(getServer(serverName)).toBeTruthy();
|
||||
expect(existsSync(join(fixture.t5,serverName))).toBe(true);
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment.state).toBe('cancelled');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -112,9 +112,9 @@ describe('native instance provisioning', () => {
|
||||
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('accepts cancellation through the provisioning API and rejects an already completed creation', async () => {
|
||||
it('accepts creation cleanup through the provisioning API and rejects an already completed creation', async () => {
|
||||
const job = await queue();
|
||||
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
|
||||
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel-creation/+server.js');
|
||||
const response = POST({ params: { id: job.deploymentId } });
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) });
|
||||
@@ -1173,3 +1173,151 @@ describe('automatic capacity admission and placement', () => {
|
||||
expect(db().prepare('SELECT port FROM native_instances').all()).toEqual([{ port: 28960 }, { port: 28960 }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('native provisioning cancellation and rollback', () => {
|
||||
it('cancels an unhealthy restart and prevents a late health response from reviving it', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const restart = requestLifecycle(job.name, 'restart'); if (!restart.ok) throw new Error(restart.error);
|
||||
await advance();
|
||||
const command = (await operationFor(restart.deploymentId))!;
|
||||
reportHostOperation(hostId, command.id, { state: 'completed' });
|
||||
let release!: (value: never) => void;
|
||||
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
||||
const pending = advance();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(POST({ params: { id: restart.deploymentId } }).status).toBe(200);
|
||||
release({ healthy: true, state: 'running' } as never);
|
||||
await pending;
|
||||
expect(await detail(restart.deploymentId)).toMatchObject({ state: 'cancelled' });
|
||||
const replacement = requestLifecycle(job.name, 'restart'); if (!replacement.ok) throw new Error(replacement.error);
|
||||
await advance();
|
||||
expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', state: 'pending' });
|
||||
});
|
||||
|
||||
it.each([false, true])('withdraws an undelivered restart or waits for a delivered one before replacement (delivered=%s)', async delivered => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const first = requestLifecycle(job.name, 'restart'); if (!first.ok) throw new Error(first.error);
|
||||
await advance(); const command = (await operationFor(first.deploymentId))!;
|
||||
if (delivered) pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(first.deploymentId)).toBe(true);
|
||||
const next = requestLifecycle(job.name, 'restart'); if (!next.ok) throw new Error(next.error);
|
||||
await advance();
|
||||
if (delivered) {
|
||||
expect(await operationFor(next.deploymentId)).toBeNull();
|
||||
reportHostOperation(hostId, command.id, { state: 'completed' });
|
||||
await advance();
|
||||
} else {
|
||||
expect((await operationFor(first.deploymentId))?.state).toBe('failed');
|
||||
expect(pendingHostOperations(hostId).some(op => op.id === command.id)).toBe(false);
|
||||
}
|
||||
expect(await operationFor(next.deploymentId)).not.toBeNull();
|
||||
expect((await detail(first.deploymentId)).state).toBe('cancelled');
|
||||
});
|
||||
|
||||
it('rolls back a completed native activation even while its restart is stuck starting', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { script, version } = createScript({ name: 'Broken beta', kind: 'script', targetRelPath: 'beta.gsc', content: Buffer.from('broken'), scopeType: 'game', scopeValue: 't5' });
|
||||
const betaId = Number(db().prepare("INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')").run(script.id,version.id).lastInsertRowid);
|
||||
db().prepare('INSERT INTO beta_scripts(beta_id,script_id,script_version_id) VALUES(?,?,?)').run(betaId,script.id,version.id);
|
||||
db().prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')").run(betaId,job.name);
|
||||
db().prepare('INSERT INTO beta_server_scripts(beta_id,server,script_id,had_original) VALUES(?,?,?,0)').run(betaId,job.name,script.id);
|
||||
queueFileChange({ sourceKind: 'beta', sourceId: String(betaId), server: job.name, kind: 'script', relPath: 'beta.gsc', content: Buffer.from('broken'), inventorySource: 'beta' });
|
||||
const restart = requestRestart(job.name,{kind:'beta',id:String(betaId)});
|
||||
await advance();
|
||||
const staging = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as {stage_operation_id:string};
|
||||
reportHostOperation(hostId,staging.stage_operation_id,{state:'completed'});
|
||||
await advance();
|
||||
const activation = (await operationFor(restart.id))!;
|
||||
const files = JSON.parse((db().prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?').get(restart.id) as {files_json:string}).files_json);
|
||||
remote.state = 'starting';
|
||||
reportHostOperation(hostId,activation.id,{state:'completed',result:{before:files.map((f:{id:number;path:string})=>({...f,contentBase64:Buffer.from('working').toString('base64')}))}});
|
||||
// Rollback must consume the receipt itself; the health worker has not caught up.
|
||||
const rollback = await rollbackBeta(betaId);
|
||||
expect((await detail(restart.id)).state).toBe('cancelled');
|
||||
const replacement = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as {provisioning_id:string};
|
||||
expect(replacement.provisioning_id).not.toBe(restart.id);
|
||||
expect(db().prepare("SELECT content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?").get(rollback.deploymentId))
|
||||
.toEqual({content:Buffer.from('working')});
|
||||
await advance();
|
||||
const stagedRollback = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(replacement.provisioning_id) as {stage_operation_id:string};
|
||||
reportHostOperation(hostId,stagedRollback.stage_operation_id,{state:'completed'});
|
||||
await advance();
|
||||
expect(await operationFor(replacement.provisioning_id)).toMatchObject({state:'pending'});
|
||||
});
|
||||
});
|
||||
|
||||
describe('cancelled native mutation bookkeeping', () => {
|
||||
it('keeps explicit cleanup available after cancelling an unsubmitted creation', async () => {
|
||||
const job = await queue();
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
|
||||
expect(canCancelCreation(job.deploymentId)).toBe(true);
|
||||
const cleanup = cancelCreation(job.deploymentId)!;
|
||||
expect((await detail(cleanup)).state).toBe('done');
|
||||
});
|
||||
|
||||
it.each(['rebuild','destroy'] as const)('reconciles a delivered %s receipt without resuming the cancelled job', async action => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const result = action === 'rebuild' ? requestLifecycle(job.name,'rebuild') : destroyServer(job.name);
|
||||
if (!result.ok || !result.deploymentId) throw new Error('Job was not queued');
|
||||
const id = result.deploymentId;
|
||||
await advance(); const operation = (await operationFor(id))!;
|
||||
pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
const replacement = action === 'rebuild' ? requestLifecycle(job.name, 'restart') : null;
|
||||
if (replacement && !replacement.ok) throw new Error(replacement.error);
|
||||
await advance();
|
||||
reportHostOperation(hostId,operation.id,{state:'completed'});
|
||||
await advance(); await advance();
|
||||
if (replacement?.ok) expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', generation: 2 });
|
||||
expect((await detail(id)).state).toBe('cancelled');
|
||||
if(action === 'rebuild') expect(db().prepare('SELECT generation,retired_at FROM native_instances WHERE server=?').get(job.name)).toEqual({generation:2,retired_at:null});
|
||||
else {
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false,true])('cancels a host migration without restarting it or leaking destination reservations (delivered=%s)', async delivered => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const destination = hosts.createHost({name:'Destination',portMin:29000,portMax:29005});
|
||||
hosts.enrollHost(hosts.createEnrollment(destination.id).token,inventory());
|
||||
updateHostLibraryCaches(destination.id,[{libraryVersionId:libraryId,status:'ready',manifestSha256:getLibraryVersion(libraryId)!.manifestSha256}]);
|
||||
hosts.updateHost(hostId,{mode:'draining'});
|
||||
await processProvisioning();
|
||||
const migration = db().prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations').get() as {deployment_id:string;remove_operation_id:string};
|
||||
expect(migration.remove_operation_id).toBeTruthy();
|
||||
if(delivered) hosts.pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(migration.deployment_id)).toBe(true);
|
||||
if(delivered) hosts.reportHostOperation(hostId,migration.remove_operation_id,{state:'completed'});
|
||||
await processProvisioning(); await processProvisioning();
|
||||
expect((await detail(migration.deployment_id)).state).toBe('cancelled');
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(destination.id)).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM ec_host_migrations').all()).toHaveLength(1);
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toHaveLength(delivered ? 0 : 2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kickProvisioning: vi.fn(async () => {}) }));
|
||||
let fixture: Fixture;
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
||||
afterEach(async () => { await fixture.cleanup(); });
|
||||
|
||||
describe('provisioning cancellation', () => {
|
||||
it.each(['create', 'start', 'restart', 'rebuild', 'stop', 'destroy'])('cancels %s in every active phase and keeps cancellation terminal', async action => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { cancelProvisioningJob, canCancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { finishDeployment, getDeployment, listDeployments } = await import('../../src/lib/server/provisioning/history.js');
|
||||
for (const state of ['queued', 'waiting', 'running', 'awaiting-health', 'interrupted']) {
|
||||
const id = `${action}-${state}`;
|
||||
db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
|
||||
VALUES(?,'server',?,?, 'In progress','2026-10-05',?,'test')`).run(id, action, state, state === 'waiting' ? null : '2026-10-05');
|
||||
expect(canCancelProvisioningJob(id)).toBe(true);
|
||||
expect(getDeployment(id)?.canCancel).toBe(true);
|
||||
expect(listDeployments().find(row => row.id === id)?.canCancel).toBe(1);
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
finishDeployment(id, { ok: true });
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'cancelled', stage: 'Cancelled by operator' });
|
||||
expect(getDeployment(id)?.canCancel).toBe(false);
|
||||
expect(db().prepare("SELECT COUNT(*) n FROM server_deployment_events WHERE deployment_id=? AND kind='outcome'").get(id)).toEqual({ n: 1 });
|
||||
}
|
||||
});
|
||||
|
||||
it('releases only the cancelled job’s recovery slot and cancels its pending file changes', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
db().prepare("INSERT INTO servers(name,komodo_name) VALUES('one','one'),('two','two')").run();
|
||||
const id = beginDeployment('one','restart');
|
||||
const other = beginDeployment('two','restart');
|
||||
db().prepare("INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES('one','2026-10-05',?),('two','2026-10-05',?)").run(id, other);
|
||||
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'campaign','source','one')").run(id);
|
||||
queueFileChange({ sourceKind: 'campaign', sourceId: 'source', server: 'one', kind: 'script', relPath: 'test.gsc', content: Buffer.from('queued'), inventorySource: 'test' });
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
expect(db().prepare('SELECT operation_id FROM restart_rollout').all()).toEqual([{ operation_id: other }]);
|
||||
expect(db().prepare('SELECT state FROM staged_file_changes').get()).toEqual({ state: 'cancelled' });
|
||||
});
|
||||
|
||||
it('rejects missing, completed and deleted-server jobs through the cancellation API', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js');
|
||||
for (const state of ['done','warning','failed']) {
|
||||
db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES(?,'server','restart',?,'Finished','2026-10-05','test')`).run(state,state);
|
||||
expect(POST({ params: { id: state } }).status).toBe(409);
|
||||
}
|
||||
expect(POST({ params: { id: 'missing' } }).status).toBe(409);
|
||||
db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by,server_deleted_at) VALUES('deleted','server','restart','running','Running','2026-10-05','test','2026-10-05')`).run();
|
||||
expect(POST({ params: { id: 'deleted' } }).status).toBe(409);
|
||||
});
|
||||
});
|
||||
@@ -535,3 +535,27 @@ describe('Watchdog provisioning and telemetry evidence', () => {
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'done' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('cancelling submitted legacy commands', () => {
|
||||
it.each(['start', 'restart', 'stop'] as const)('ignores a late %s response after cancellation', async action => {
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let release!: (value: { ok: true }) => void;
|
||||
const command = action === 'restart' ? komodo.komodoRestart : komodo.komodoRuntimeAction;
|
||||
vi.mocked(command).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
||||
const id = await queue(action);
|
||||
const pending = processProvisioning();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
release({ ok: true });
|
||||
await pending;
|
||||
expect(await operation(id)).toMatchObject({ state: 'cancelled', stage: 'Cancelled by operator' });
|
||||
expect(db().prepare('SELECT * FROM restart_rollout WHERE operation_id=?').all(id)).toEqual([]);
|
||||
const replacement = await queue(action);
|
||||
await processProvisioning();
|
||||
expect(await operation(replacement)).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(command).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -395,7 +395,7 @@ describe('shared activation and partial writes', () => {
|
||||
.get('legacy-drift') as { provisioning_id: string; restart_state: string; error: string };
|
||||
const restoration = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?')
|
||||
.get(rollback.deploymentId) as { provisioning_id: string };
|
||||
expect(old).toMatchObject({ restart_state: 'failed', error: expect.stringContaining('changed while queued') });
|
||||
expect(old).toMatchObject({ restart_state: 'failed', error: expect.stringContaining('Superseded by rollback') });
|
||||
expect(restoration.provisioning_id).not.toBe(old.provisioning_id);
|
||||
});
|
||||
|
||||
@@ -438,3 +438,26 @@ describe('shared activation and partial writes', () => {
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollback supersedes provisioning', () => {
|
||||
it.each(['when-empty', 'manual'] as const)('replaces an unhealthy beta restart with a %s rollback without waiting for recovery', async restart => {
|
||||
const beta = await queueChange('beta');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await tick('beta');
|
||||
const old = db().prepare('SELECT provisioning_id FROM beta_servers WHERE beta_id=?').get(beta.sourceId) as { provisioning_id: string };
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(old.provisioning_id)).toEqual({ state: 'awaiting-health' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'container failed to start' });
|
||||
const rollback = await rollbackBeta(Number(beta.sourceId), { restart });
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(old.provisioning_id)).toEqual({ state: 'cancelled' });
|
||||
expect(db().prepare('SELECT * FROM restart_rollout WHERE operation_id=?').all(old.provisioning_id)).toEqual([]);
|
||||
const replacement = db().prepare('SELECT provisioning_id,restart_state FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as { provisioning_id: string; restart_state: string };
|
||||
expect(replacement.provisioning_id).not.toBe(old.provisioning_id);
|
||||
await tick('campaign');
|
||||
expect(await readFile(destination())).toEqual(restart === 'manual' ? after : before);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(restart === 'manual' ? 1 : 2);
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(replacement.provisioning_id))
|
||||
.toEqual({ state: restart === 'manual' ? 'waiting' : 'awaiting-health' });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user