Fix rebuild generation reconciliation (Refs #155)
This commit is contained in:
@@ -465,7 +465,16 @@ describe('provisioning health handoff', () => {
|
||||
.run(old, old, old, job.name);
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
const observedAt = new Date().toISOString();
|
||||
if (action === 'rebuild') {
|
||||
// A new generation now establishes its own durable evidence boundary.
|
||||
// Observe game health strictly after that transition.
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
vi.mocked(nativeRequest).mockImplementationOnce(async () => {
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
return { healthy: true, state: 'running', exists: true } as never;
|
||||
});
|
||||
}
|
||||
const observedAt = new Date(Date.now() + (action === 'rebuild' ? 1 : 0)).toISOString();
|
||||
const roster = [
|
||||
{ num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }
|
||||
];
|
||||
@@ -4288,3 +4297,276 @@ describe('game key pool launch lifecycle', () => {
|
||||
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('completed rebuild generation receipts (#155)', () => {
|
||||
async function rebuild() {
|
||||
const job = await queue();
|
||||
await advance();
|
||||
await acknowledge(job.deploymentId);
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const requested = requestLifecycle(job.name, 'rebuild');
|
||||
if (!requested.ok) throw new Error(requested.error);
|
||||
await advance();
|
||||
return {
|
||||
job,
|
||||
id: requested.deploymentId,
|
||||
operation: (await operationFor(requested.deploymentId))!
|
||||
};
|
||||
}
|
||||
|
||||
async function inventoryGeneration(
|
||||
instanceId: string,
|
||||
generation: number,
|
||||
observedAt = new Date().toISOString(),
|
||||
targetHost = hostId
|
||||
) {
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(targetHost, {
|
||||
...inventory(),
|
||||
instanceStates: [
|
||||
{ instanceId, generation, state: 'running', healthy: true, observedAt }
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
it('repairs an older job binding from the actual command before verification, including duplicate receipts and restart', async () => {
|
||||
const { job, id } = await rebuild();
|
||||
await acknowledge(id);
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const next = requestLifecycle(job.name, 'rebuild');
|
||||
if (!next.ok) throw new Error(next.error);
|
||||
await advance();
|
||||
const operation = (await operationFor(next.deploymentId))!;
|
||||
expect(operation.generation).toBe(2);
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
// Historical job snapshots could stay on generation 1 while the submitted
|
||||
// command correctly targeted 2 -> 3. The old completion path kept 2.
|
||||
db()
|
||||
.prepare('UPDATE native_provisioning SET generation=1 WHERE deployment_id=?')
|
||||
.run(next.deploymentId);
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, operation.id, {
|
||||
state: 'completed',
|
||||
result: { instanceId: operation.instanceId, generation: 3, healthy: true }
|
||||
});
|
||||
await inventoryGeneration(operation.instanceId!, 3);
|
||||
closeDb();
|
||||
const { nativeRequest, nativeInstance } =
|
||||
await import('../../src/lib/server/backends/native.js');
|
||||
vi.mocked(nativeRequest).mockClear();
|
||||
remote.state = 'starting';
|
||||
await advance();
|
||||
expect(nativeInstance(job.name)?.generation).toBe(3);
|
||||
expect(nativeRequest).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: operation.instanceId, generation: 3 }),
|
||||
'instance.inspect'
|
||||
);
|
||||
expect(
|
||||
db()
|
||||
.prepare(
|
||||
'SELECT generation,applied_at FROM native_provisioning WHERE deployment_id=?'
|
||||
)
|
||||
.get(next.deploymentId)
|
||||
).toEqual({ generation: 2, applied_at: expect.any(String) });
|
||||
const { reconcileNativeProvisioningReceipt } =
|
||||
await import('../../src/lib/server/provisioning/native.js');
|
||||
const before = db()
|
||||
.prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
|
||||
.get(operation.instanceId);
|
||||
reconcileNativeProvisioningReceipt(next.deploymentId);
|
||||
closeDb();
|
||||
reconcileNativeProvisioningReceipt(next.deploymentId);
|
||||
expect(nativeInstance(job.name)?.generation).toBe(3);
|
||||
expect(
|
||||
db()
|
||||
.prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
|
||||
.get(operation.instanceId)
|
||||
).toEqual(before);
|
||||
expect((await detail(next.deploymentId)).state).toBe('awaiting-health');
|
||||
for (const observation of await observations(job.name))
|
||||
expect(observation.health).not.toBe('online');
|
||||
const { latestGameSubmission, latestGameSubmissions } =
|
||||
await import('../../src/lib/server/fleet/lifecycle-evidence.js');
|
||||
expect(latestGameSubmission(job.name)).toBe(
|
||||
(before as { generation_reconciled_at: string }).generation_reconciled_at
|
||||
);
|
||||
expect(latestGameSubmissions().get(job.name)).toBe(latestGameSubmission(job.name));
|
||||
remote.state = 'running';
|
||||
await advance();
|
||||
expect((await detail(next.deploymentId)).state).toBe('done');
|
||||
const restart = requestLifecycle(job.name, 'restart');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await advance();
|
||||
expect(await operationFor(restart.deploymentId)).toMatchObject({ generation: 3 });
|
||||
});
|
||||
|
||||
it.each(['before', 'after'] as const)(
|
||||
'reconciles delayed completion reported %s cancellation across restart',
|
||||
async (when) => {
|
||||
const { job, id, operation } = await rebuild();
|
||||
const { reportHostOperation, pendingHostOperations } =
|
||||
await import('../../src/lib/server/hosts/index.js');
|
||||
const { cancelProvisioningJob } =
|
||||
await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
pendingHostOperations(hostId);
|
||||
if (when === 'before')
|
||||
reportHostOperation(hostId, operation.id, { state: 'completed' });
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
closeDb();
|
||||
if (when === 'after') reportHostOperation(hostId, operation.id, { state: 'completed' });
|
||||
await advance();
|
||||
await advance();
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
expect(nativeInstance(job.name)?.generation).toBe(2);
|
||||
expect((await detail(id)).state).toBe('cancelled');
|
||||
}
|
||||
);
|
||||
|
||||
it('recovers an already-applied cancelled mismatch through the API and targets replacement logs, health and removal', async () => {
|
||||
const { job, id, operation } = await rebuild();
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, operation.id, {
|
||||
state: 'completed',
|
||||
result: { instanceId: operation.instanceId, generation: 2 }
|
||||
});
|
||||
const { cancelProvisioningJob } =
|
||||
await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db()
|
||||
.prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?')
|
||||
.run(operation.updatedAt, id);
|
||||
await inventoryGeneration(operation.instanceId!, 2);
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
expect(getDeployment(id)?.generationRecovery).toMatchObject({
|
||||
expectedGeneration: 1,
|
||||
observedGeneration: 2,
|
||||
available: true
|
||||
});
|
||||
const { POST } =
|
||||
await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
|
||||
expect(await POST({ params: { id } }).json()).toEqual({ ok: true, generation: 2 });
|
||||
expect(POST({ params: { id } }).status).toBe(200);
|
||||
expect((await detail(id)).state).toBe('cancelled');
|
||||
expect(getDeployment(id)?.generationRecovery).toBeNull();
|
||||
const backend = await import('../../src/lib/server/backends/native.js');
|
||||
vi.mocked(backend.nativeRequest).mockRestore();
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
vi.spyOn(hosts, 'waitHostOperationStatus').mockImplementation(async (id) =>
|
||||
hosts.reportHostOperationStatus(hostId, id, {
|
||||
state: 'completed',
|
||||
result: { healthy: true, state: 'running' }
|
||||
})
|
||||
);
|
||||
await backend.nativeRequest(backend.nativeInstance(job.name)!, 'logs.read', {
|
||||
file: 'console.log'
|
||||
});
|
||||
expect(await backend.nativeHealth(job.name)).toMatchObject({ ok: true });
|
||||
expect(
|
||||
db()
|
||||
.prepare(
|
||||
"SELECT generation FROM ec_host_operations WHERE kind IN ('logs.read','instance.inspect')"
|
||||
)
|
||||
.all()
|
||||
.every((row) => (row as { generation: number }).generation === 2)
|
||||
).toBe(true);
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const removal = destroyServer(job.name);
|
||||
if (!removal.ok || !removal.deploymentId) throw new Error('Removal not queued');
|
||||
await advance();
|
||||
expect(await operationFor(removal.deploymentId)).toMatchObject({
|
||||
generation: 2,
|
||||
kind: 'instance.remove'
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
'host',
|
||||
'instance',
|
||||
'result-instance',
|
||||
'result-generation',
|
||||
'payload',
|
||||
'stale-inventory',
|
||||
'missing-inventory',
|
||||
'inventory-instance'
|
||||
] as const)('rejects recovery evidence with %s mismatch', async (mismatch) => {
|
||||
const { job, id, operation } = await rebuild();
|
||||
const { reportHostOperation, createHost } =
|
||||
await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, operation.id, {
|
||||
state: 'completed',
|
||||
result: {
|
||||
instanceId:
|
||||
mismatch === 'result-instance' ? 'another-instance' : operation.instanceId,
|
||||
generation: mismatch === 'result-generation' ? 3 : 2
|
||||
}
|
||||
});
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
if (mismatch === 'host')
|
||||
db()
|
||||
.prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?')
|
||||
.run(createHost({ name: 'Other host' }).id, operation.id);
|
||||
if (mismatch === 'instance')
|
||||
db()
|
||||
.prepare('UPDATE ec_host_operations SET instance_id=? WHERE id=?')
|
||||
.run('another-instance', operation.id);
|
||||
if (mismatch === 'payload')
|
||||
db()
|
||||
.prepare(
|
||||
"UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?"
|
||||
)
|
||||
.run(operation.id);
|
||||
if (mismatch !== 'missing-inventory')
|
||||
await inventoryGeneration(
|
||||
mismatch === 'inventory-instance' ? 'another-instance' : operation.instanceId!,
|
||||
2,
|
||||
mismatch === 'stale-inventory' ? '2000-01-01T00:00:00Z' : new Date().toISOString()
|
||||
);
|
||||
const { POST } =
|
||||
await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
|
||||
expect(POST({ params: { id } }).status).toBe(409);
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
expect(nativeInstance(job.name)?.generation).toBe(1);
|
||||
});
|
||||
|
||||
it('pauses after an agent stale-generation rejection even when inventory is unavailable', async () => {
|
||||
const { id, operation } = await rebuild();
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, operation.id, { state: 'completed' });
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
vi.mocked(nativeRequest).mockClear();
|
||||
vi.mocked(nativeRequest).mockRejectedValueOnce(
|
||||
new Error('Instance generation has been superseded or removed')
|
||||
);
|
||||
await advance();
|
||||
expect((await detail(id)).state).toBe('interrupted');
|
||||
expect((await detail(id)).error).toContain('controller expects 2, host reports unknown');
|
||||
await advance();
|
||||
expect(nativeRequest).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('stops verification retries on a confirmed generation mismatch and keeps expected/observed evidence', async () => {
|
||||
const { id, operation } = await rebuild();
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, operation.id, { state: 'completed' });
|
||||
await inventoryGeneration(operation.instanceId!, 3);
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
vi.mocked(nativeRequest).mockClear();
|
||||
await advance();
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const before = getDeployment(id)!;
|
||||
expect(before.deployment.state).toBe('interrupted');
|
||||
expect(before.progress?.blocker).toMatchObject({
|
||||
code: 'generation-recovery',
|
||||
automatic: false,
|
||||
message: expect.stringContaining(
|
||||
'controller expects 1, completed rebuild requests 2, host reports 3'
|
||||
)
|
||||
});
|
||||
await advance();
|
||||
await advance();
|
||||
expect(nativeRequest).not.toHaveBeenCalled();
|
||||
expect(getDeployment(id)?.events).toEqual(before.events);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user