Cancel active provisioning jobs when rolling back betas
This commit is contained in:
@@ -112,9 +112,9 @@ describe('native instance provisioning', () => {
|
||||
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('accepts cancellation through the provisioning API and rejects an already completed creation', async () => {
|
||||
it('accepts creation cleanup through the provisioning API and rejects an already completed creation', async () => {
|
||||
const job = await queue();
|
||||
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
|
||||
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel-creation/+server.js');
|
||||
const response = POST({ params: { id: job.deploymentId } });
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) });
|
||||
@@ -1173,3 +1173,151 @@ describe('automatic capacity admission and placement', () => {
|
||||
expect(db().prepare('SELECT port FROM native_instances').all()).toEqual([{ port: 28960 }, { port: 28960 }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('native provisioning cancellation and rollback', () => {
|
||||
it('cancels an unhealthy restart and prevents a late health response from reviving it', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const restart = requestLifecycle(job.name, 'restart'); if (!restart.ok) throw new Error(restart.error);
|
||||
await advance();
|
||||
const command = (await operationFor(restart.deploymentId))!;
|
||||
reportHostOperation(hostId, command.id, { state: 'completed' });
|
||||
let release!: (value: never) => void;
|
||||
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
||||
const pending = advance();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(POST({ params: { id: restart.deploymentId } }).status).toBe(200);
|
||||
release({ healthy: true, state: 'running' } as never);
|
||||
await pending;
|
||||
expect(await detail(restart.deploymentId)).toMatchObject({ state: 'cancelled' });
|
||||
const replacement = requestLifecycle(job.name, 'restart'); if (!replacement.ok) throw new Error(replacement.error);
|
||||
await advance();
|
||||
expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', state: 'pending' });
|
||||
});
|
||||
|
||||
it.each([false, true])('withdraws an undelivered restart or waits for a delivered one before replacement (delivered=%s)', async delivered => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const first = requestLifecycle(job.name, 'restart'); if (!first.ok) throw new Error(first.error);
|
||||
await advance(); const command = (await operationFor(first.deploymentId))!;
|
||||
if (delivered) pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(first.deploymentId)).toBe(true);
|
||||
const next = requestLifecycle(job.name, 'restart'); if (!next.ok) throw new Error(next.error);
|
||||
await advance();
|
||||
if (delivered) {
|
||||
expect(await operationFor(next.deploymentId)).toBeNull();
|
||||
reportHostOperation(hostId, command.id, { state: 'completed' });
|
||||
await advance();
|
||||
} else {
|
||||
expect((await operationFor(first.deploymentId))?.state).toBe('failed');
|
||||
expect(pendingHostOperations(hostId).some(op => op.id === command.id)).toBe(false);
|
||||
}
|
||||
expect(await operationFor(next.deploymentId)).not.toBeNull();
|
||||
expect((await detail(first.deploymentId)).state).toBe('cancelled');
|
||||
});
|
||||
|
||||
it('rolls back a completed native activation even while its restart is stuck starting', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { script, version } = createScript({ name: 'Broken beta', kind: 'script', targetRelPath: 'beta.gsc', content: Buffer.from('broken'), scopeType: 'game', scopeValue: 't5' });
|
||||
const betaId = Number(db().prepare("INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')").run(script.id,version.id).lastInsertRowid);
|
||||
db().prepare('INSERT INTO beta_scripts(beta_id,script_id,script_version_id) VALUES(?,?,?)').run(betaId,script.id,version.id);
|
||||
db().prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')").run(betaId,job.name);
|
||||
db().prepare('INSERT INTO beta_server_scripts(beta_id,server,script_id,had_original) VALUES(?,?,?,0)').run(betaId,job.name,script.id);
|
||||
queueFileChange({ sourceKind: 'beta', sourceId: String(betaId), server: job.name, kind: 'script', relPath: 'beta.gsc', content: Buffer.from('broken'), inventorySource: 'beta' });
|
||||
const restart = requestRestart(job.name,{kind:'beta',id:String(betaId)});
|
||||
await advance();
|
||||
const staging = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as {stage_operation_id:string};
|
||||
reportHostOperation(hostId,staging.stage_operation_id,{state:'completed'});
|
||||
await advance();
|
||||
const activation = (await operationFor(restart.id))!;
|
||||
const files = JSON.parse((db().prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?').get(restart.id) as {files_json:string}).files_json);
|
||||
remote.state = 'starting';
|
||||
reportHostOperation(hostId,activation.id,{state:'completed',result:{before:files.map((f:{id:number;path:string})=>({...f,contentBase64:Buffer.from('working').toString('base64')}))}});
|
||||
// Rollback must consume the receipt itself; the health worker has not caught up.
|
||||
const rollback = await rollbackBeta(betaId);
|
||||
expect((await detail(restart.id)).state).toBe('cancelled');
|
||||
const replacement = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as {provisioning_id:string};
|
||||
expect(replacement.provisioning_id).not.toBe(restart.id);
|
||||
expect(db().prepare("SELECT content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?").get(rollback.deploymentId))
|
||||
.toEqual({content:Buffer.from('working')});
|
||||
await advance();
|
||||
const stagedRollback = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(replacement.provisioning_id) as {stage_operation_id:string};
|
||||
reportHostOperation(hostId,stagedRollback.stage_operation_id,{state:'completed'});
|
||||
await advance();
|
||||
expect(await operationFor(replacement.provisioning_id)).toMatchObject({state:'pending'});
|
||||
});
|
||||
});
|
||||
|
||||
describe('cancelled native mutation bookkeeping', () => {
|
||||
it('keeps explicit cleanup available after cancelling an unsubmitted creation', async () => {
|
||||
const job = await queue();
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
|
||||
expect(canCancelCreation(job.deploymentId)).toBe(true);
|
||||
const cleanup = cancelCreation(job.deploymentId)!;
|
||||
expect((await detail(cleanup)).state).toBe('done');
|
||||
});
|
||||
|
||||
it.each(['rebuild','destroy'] as const)('reconciles a delivered %s receipt without resuming the cancelled job', async action => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const result = action === 'rebuild' ? requestLifecycle(job.name,'rebuild') : destroyServer(job.name);
|
||||
if (!result.ok || !result.deploymentId) throw new Error('Job was not queued');
|
||||
const id = result.deploymentId;
|
||||
await advance(); const operation = (await operationFor(id))!;
|
||||
pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
const replacement = action === 'rebuild' ? requestLifecycle(job.name, 'restart') : null;
|
||||
if (replacement && !replacement.ok) throw new Error(replacement.error);
|
||||
await advance();
|
||||
reportHostOperation(hostId,operation.id,{state:'completed'});
|
||||
await advance(); await advance();
|
||||
if (replacement?.ok) expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', generation: 2 });
|
||||
expect((await detail(id)).state).toBe('cancelled');
|
||||
if(action === 'rebuild') expect(db().prepare('SELECT generation,retired_at FROM native_instances WHERE server=?').get(job.name)).toEqual({generation:2,retired_at:null});
|
||||
else {
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false,true])('cancels a host migration without restarting it or leaking destination reservations (delivered=%s)', async delivered => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const destination = hosts.createHost({name:'Destination',portMin:29000,portMax:29005});
|
||||
hosts.enrollHost(hosts.createEnrollment(destination.id).token,inventory());
|
||||
updateHostLibraryCaches(destination.id,[{libraryVersionId:libraryId,status:'ready',manifestSha256:getLibraryVersion(libraryId)!.manifestSha256}]);
|
||||
hosts.updateHost(hostId,{mode:'draining'});
|
||||
await processProvisioning();
|
||||
const migration = db().prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations').get() as {deployment_id:string;remove_operation_id:string};
|
||||
expect(migration.remove_operation_id).toBeTruthy();
|
||||
if(delivered) hosts.pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(migration.deployment_id)).toBe(true);
|
||||
if(delivered) hosts.reportHostOperation(hostId,migration.remove_operation_id,{state:'completed'});
|
||||
await processProvisioning(); await processProvisioning();
|
||||
expect((await detail(migration.deployment_id)).state).toBe('cancelled');
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(destination.id)).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM ec_host_migrations').all()).toHaveLength(1);
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toHaveLength(delivered ? 0 : 2);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user