319 lines
14 KiB
TypeScript
319 lines
14 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { prepareNativeHost } from '../helpers/native-host.js';
|
|
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
|
|
|
const server = 't5_zombie_theater_1';
|
|
const campaign = 'recover-campaign';
|
|
const started = '2026-10-19T18:38:00.000Z';
|
|
const failedAt = '2026-10-19T18:48:42.000Z';
|
|
const recoveredAt = '2026-10-19T21:31:00.000Z';
|
|
let fixture: Fixture;
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(started);
|
|
fixture = await newFixture([server]);
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
await prepareNativeHost([server]);
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
async function failedCampaign() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const handle = db();
|
|
handle.prepare('UPDATE native_instances SET generation=5').run();
|
|
handle
|
|
.prepare("INSERT INTO deployments(id,action,started_at) VALUES (?,'promote',?)")
|
|
.run(campaign, started);
|
|
handle
|
|
.prepare(
|
|
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,?,2,'failed')"
|
|
)
|
|
.run(campaign, server);
|
|
for (const path of ['common.gsc', 'leaderboard.gsc']) {
|
|
queueFileChange({
|
|
sourceKind: 'campaign',
|
|
sourceId: campaign,
|
|
server,
|
|
kind: 'script',
|
|
relPath: path,
|
|
content: Buffer.from(`intended ${path}`),
|
|
inventorySource: campaign
|
|
});
|
|
}
|
|
const failed = requestRestart(server, { kind: 'campaign', id: campaign });
|
|
vi.setSystemTime(failedAt);
|
|
finishDeployment(failed.id, { ok: false, message: 'Artifact download timed out' });
|
|
return { handle, failed };
|
|
}
|
|
|
|
async function recoveredCampaign() {
|
|
const { handle, failed } = await failedCampaign();
|
|
const { beginDeployment, finishDeployment } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
const { captureRecipePlan, recipeStorage } =
|
|
await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
vi.setSystemTime('2026-10-19T21:29:00.000Z');
|
|
const recovery = beginDeployment(server, 'rebuild');
|
|
const instance = nativeInstance(server)!;
|
|
const files = handle
|
|
.prepare('SELECT kind,rel_path,content_hash FROM staged_file_changes WHERE source_id=?')
|
|
.all(campaign) as { kind: 'script'; rel_path: string; content_hash: string }[];
|
|
const recipe = captureRecipePlan('t5', 'theater');
|
|
for (const file of files) {
|
|
recipe.files.push({
|
|
kind: file.kind,
|
|
path: file.rel_path,
|
|
sha256: file.content_hash,
|
|
name: file.rel_path,
|
|
size: 10,
|
|
versionId: null,
|
|
version: null,
|
|
releaseVersion: null
|
|
});
|
|
recordScript(server, file.rel_path, null, file.content_hash, 10, 'native-create');
|
|
}
|
|
const stored = recipeStorage(recipe);
|
|
handle
|
|
.prepare(
|
|
'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES (?,?,?)'
|
|
)
|
|
.run(recovery, stored.recipe, stored.recipeSummary);
|
|
handle
|
|
.prepare(
|
|
`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,payload,state,created_at,updated_at)
|
|
VALUES ('replacement',?,'instance.rebuild',?,5,'{"nextGeneration":6}','completed',?,?)`
|
|
)
|
|
.run(instance.host_id, instance.id, new Date().toISOString(), recoveredAt);
|
|
handle
|
|
.prepare(
|
|
'UPDATE native_provisioning SET operation_id=?,applied_at=?,verified_at=? WHERE deployment_id=?'
|
|
)
|
|
.run('replacement', recoveredAt, recoveredAt, recovery);
|
|
handle.prepare('UPDATE native_instances SET generation=6 WHERE id=?').run(instance.id);
|
|
handle
|
|
.prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?')
|
|
.run(new Date().toISOString(), recovery);
|
|
vi.setSystemTime(recoveredAt);
|
|
finishDeployment(recovery, { ok: true });
|
|
return { handle, failed, recovery, recipe, instance };
|
|
}
|
|
|
|
async function sweep() {
|
|
return (await import('../../src/lib/server/pipeline/sweep.js')).sweep();
|
|
}
|
|
|
|
describe('campaign recovery', () => {
|
|
it('adopts a verified matching rebuild, completes the campaign, and retains both operations and failure history', async () => {
|
|
const { handle, failed, recovery } = await recoveredCampaign();
|
|
const original = handle
|
|
.prepare('SELECT * FROM server_deployments WHERE id=?')
|
|
.get(failed.id);
|
|
const events = handle
|
|
.prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?')
|
|
.all(failed.id);
|
|
const { fleetOperations, serverOperations } =
|
|
await import('../../src/lib/server/fleet/overview.js');
|
|
expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({
|
|
state: 'attention',
|
|
evidenceAt: failedAt
|
|
});
|
|
expect(await sweep()).toMatchObject({ healthy: 1, failed: 0 });
|
|
expect(
|
|
handle
|
|
.prepare(
|
|
'SELECT restart_state,provisioning_id,error,checked_at,restarted_at FROM deployment_servers'
|
|
)
|
|
.get()
|
|
).toEqual({
|
|
restart_state: 'done',
|
|
provisioning_id: recovery,
|
|
error: null,
|
|
checked_at: recoveredAt,
|
|
restarted_at: '2026-10-19T21:29:00.000Z'
|
|
});
|
|
expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
|
|
finished_at: recoveredAt
|
|
});
|
|
expect(fleetOperations()).toEqual([]);
|
|
expect(serverOperations(server).find((op) => op.kind === 'campaign')).toMatchObject({
|
|
state: 'done',
|
|
evidenceAt: recoveredAt
|
|
});
|
|
expect(
|
|
handle.prepare('SELECT * FROM server_deployments WHERE id=?').get(failed.id)
|
|
).toEqual(original);
|
|
expect(
|
|
handle
|
|
.prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?')
|
|
.all(failed.id)
|
|
).toEqual(events);
|
|
expect(
|
|
handle
|
|
.prepare(
|
|
'SELECT operation_id FROM provisioning_sources WHERE source_id=? ORDER BY rowid'
|
|
)
|
|
.all(campaign)
|
|
).toEqual([{ operation_id: failed.id }, { operation_id: recovery }]);
|
|
expect(handle.prepare('SELECT state FROM staged_file_changes').all()).toEqual([
|
|
{ state: 'applied' },
|
|
{ state: 'applied' }
|
|
]);
|
|
const { GET } = await import('../../src/routes/api/campaigns/[id]/+server.js');
|
|
expect((await (await GET({ params: { id: campaign } })).json()).operations).toEqual([
|
|
expect.objectContaining({ id: failed.id, state: 'failed', finished_at: failedAt }),
|
|
expect.objectContaining({ id: recovery, state: 'done', finished_at: recoveredAt })
|
|
]);
|
|
await sweep();
|
|
expect(
|
|
handle
|
|
.prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'")
|
|
.all()
|
|
).toHaveLength(1);
|
|
});
|
|
|
|
it.each([
|
|
'recipe mismatch',
|
|
'inventory mismatch',
|
|
'missing recipe',
|
|
'missing file',
|
|
'stale generation',
|
|
'wrong instance',
|
|
'wrong host',
|
|
'unverified',
|
|
'failed receipt',
|
|
'unrelated restart',
|
|
'older rebuild',
|
|
'cancelled files',
|
|
'deleted target',
|
|
'staged override mismatch'
|
|
])('keeps attention for %s', async (condition) => {
|
|
const { handle, failed, recovery, recipe } = await recoveredCampaign();
|
|
if (condition === 'recipe mismatch') {
|
|
recipe.files.find((f) => f.path === 'leaderboard.gsc')!.sha256 = 'other';
|
|
handle
|
|
.prepare('UPDATE server_launch_recipes SET recipe_summary=?')
|
|
.run(JSON.stringify(recipe));
|
|
}
|
|
if (condition === 'inventory mismatch')
|
|
handle
|
|
.prepare(
|
|
"UPDATE server_scripts SET sha256='other' WHERE rel_path='leaderboard.gsc'"
|
|
)
|
|
.run();
|
|
if (condition === 'missing recipe')
|
|
handle.prepare('DELETE FROM server_launch_recipes').run();
|
|
if (condition === 'missing file')
|
|
handle
|
|
.prepare("DELETE FROM staged_file_changes WHERE rel_path='leaderboard.gsc'")
|
|
.run();
|
|
if (condition === 'stale generation')
|
|
handle.prepare('UPDATE native_instances SET generation=7').run();
|
|
if (condition === 'wrong instance')
|
|
handle.prepare("UPDATE native_instances SET server='replacement-name'").run();
|
|
if (condition === 'wrong host')
|
|
handle.prepare("UPDATE native_instances SET host_id='other-host'").run();
|
|
if (condition === 'unverified')
|
|
handle
|
|
.prepare('UPDATE native_provisioning SET verified_at=NULL WHERE deployment_id=?')
|
|
.run(recovery);
|
|
if (condition === 'failed receipt')
|
|
handle.prepare("UPDATE ec_host_operations SET state='failed'").run();
|
|
if (condition === 'unrelated restart') {
|
|
handle
|
|
.prepare("UPDATE server_deployments SET action='restart' WHERE id=?")
|
|
.run(recovery);
|
|
handle.prepare("UPDATE ec_host_operations SET kind='instance.restart'").run();
|
|
handle.prepare('UPDATE native_instances SET generation=5').run();
|
|
}
|
|
if (condition === 'older rebuild')
|
|
handle
|
|
.prepare('UPDATE server_deployments SET started_at=? WHERE id=?')
|
|
.run(started, recovery);
|
|
if (condition === 'cancelled files')
|
|
handle.prepare("UPDATE staged_file_changes SET state='cancelled'").run();
|
|
if (condition === 'deleted target')
|
|
handle.prepare('UPDATE deployment_servers SET server_deleted_at=?').run(recoveredAt);
|
|
if (condition === 'staged override mismatch')
|
|
handle.prepare('UPDATE native_provisioning SET files_json=? WHERE deployment_id=?').run(
|
|
JSON.stringify([
|
|
{
|
|
path: 'runtime/plutonium/storage/t5/raw/scripts/sp/leaderboard.gsc',
|
|
sha256: 'other'
|
|
}
|
|
]),
|
|
recovery
|
|
);
|
|
await sweep();
|
|
expect(
|
|
handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get()
|
|
).toEqual({ restart_state: 'failed', provisioning_id: failed.id });
|
|
expect(
|
|
handle
|
|
.prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'")
|
|
.all()
|
|
).toEqual([]);
|
|
if (condition !== 'deleted target') {
|
|
expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
|
|
finished_at: null
|
|
});
|
|
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
|
|
expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({
|
|
state: 'attention',
|
|
evidenceAt: failedAt
|
|
});
|
|
}
|
|
});
|
|
|
|
it('does not accept fresh healthy status without a successful replacement operation', async () => {
|
|
const { handle, failed } = await failedCampaign();
|
|
handle
|
|
.prepare(
|
|
"UPDATE servers SET last_seen_at=?,last_checked_at=?,last_error=NULL,container_state='running'"
|
|
)
|
|
.run(recoveredAt, recoveredAt);
|
|
await sweep();
|
|
expect(
|
|
handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get()
|
|
).toEqual({ restart_state: 'failed', provisioning_id: failed.id });
|
|
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
|
|
expect(
|
|
fleetOperations()
|
|
.filter((op) => op.state === 'attention')
|
|
.map((op) => op.evidenceAt)
|
|
).toEqual([failedAt, failedAt]);
|
|
});
|
|
|
|
it('waits for every remaining target before completing the campaign', async () => {
|
|
const { handle } = await recoveredCampaign();
|
|
handle
|
|
.prepare(
|
|
"INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,'other',1,'held')"
|
|
)
|
|
.run(campaign);
|
|
await sweep();
|
|
expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
|
|
finished_at: null
|
|
});
|
|
handle
|
|
.prepare("UPDATE deployment_servers SET restart_state='done' WHERE server='other'")
|
|
.run();
|
|
await sweep();
|
|
expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
|
|
finished_at: recoveredAt
|
|
});
|
|
});
|
|
});
|