josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild beta and campaign file activations

Browse Source
This commit is contained in:
josh
2026-10-12 22:52:58 -04:00
parent fa0baa8db5
commit 537b9841a7
22 changed files with 533 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/servers.ts
+5 -2
View File
@@ -190,14 +190,17 @@ export function listServers(): ServerRow[] {
}
/** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */
export function activeBetaRemovalBlocker(server: string): string | null {
export function activeBetaRemovalBlocker(server: string, activationId?: string): string | null {
const betas = db()
.prepare(
`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id=?
AND s.kind='beta' AND s.source_id=CAST(b.id AS TEXT) AND s.server=bs.server
AND bs.provisioning_id=s.operation_id)
ORDER BY b.id`
)
.all(server) as { id: number }[];
.all(server, activationId ?? null) as { id: number }[];
if (!betas.length) return null;
return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map((beta) => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`;
}
src/lib/server/pipeline/beta-plan-runner.ts
+2 -2
View File
@@ -56,11 +56,11 @@ export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void
if (!result.deploymentId)
activity(id, 'Cancelled queued changes; no deployed files need restoration');
else if (plan.rollbackRestart === 'when-empty')
activity(id, 'Queued file restoration and rolling restarts; waiting for empty servers');
activity(id, 'Queued file restoration and rolling rebuilds; waiting for empty servers');
else
activity(
id,
'File restoration is held; resume the rollback campaign to apply it with a safe restart'
'File restoration is held; resume the rollback campaign to apply it with a safe rebuild'
);
if (result.warnings.length) activity(id, result.warnings.join('; '));
}
src/lib/server/provisioning/activation-files.ts
+44
View File
@@ -0,0 +1,44 @@
import { db } from '../db/index.js';
import { blobSize, getFileBlob } from '../db/file-blobs.js';
import type { LaunchRecipePlan } from '../fleet/launch-recipes.js';
import type { StagedFileChange } from '../fleet/staged-files.js';
/** Build the replacement with the exact requested additions, updates and removals. */
export function applyActivationFiles(
recipe: LaunchRecipePlan,
operationId: string,
includeContent = false
): void {
const changes = db()
.prepare(
`SELECT f.*, ${blobSize('f.content_hash')} AS size, v.version FROM staged_file_changes f
JOIN provisioning_sources s ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
LEFT JOIN script_versions v ON v.id=f.script_version_id
WHERE s.operation_id=? AND f.state='pending' ORDER BY f.id`
)
.all(operationId) as (StagedFileChange & {
content_hash: string | null;
size: number;
version: number | null;
})[];
for (const change of changes) {
if (change.kind !== 'script' && change.kind !== 'config' && change.kind !== 'plugin')
throw new Error('File activation requires a runtime file kind.');
recipe.files = recipe.files.filter(
(file) => file.kind !== change.kind || file.path !== change.rel_path
);
if (!change.content_hash) continue;
recipe.files.push({
kind: change.kind,
name: change.rel_path,
path: change.rel_path,
versionId: change.script_version_id,
version: change.version,
sha256: change.content_hash,
size: change.size,
...(includeContent
? { content: getFileBlob(change.content_hash)!.toString('base64') }
: {})
});
}
}
src/lib/server/provisioning/launch-preparation.ts
+2
View File
@@ -19,6 +19,8 @@ export interface NativeLaunchInput {
unhealthyRecovery?: boolean;
/** Rebalance replays recorded deployed bytes instead of current promoted files. */
frozenRecipeId?: string;
/** Pending beta/campaign files override promoted launch files. */
activationId?: string;
}
export interface PreparedNativeLaunch {
src/lib/server/provisioning/launch-worker.ts
+10 -8
View File
@@ -1,5 +1,6 @@
import { parentPort } from 'node:worker_threads';
import { posix } from 'node:path';
import { applyActivationFiles } from './activation-files.js';
import { db } from '../db/index.js';
import { relDestForKind } from '../config.js';
import {
@@ -32,15 +33,15 @@ parentPort!.on(
const input = message.input;
// All file versions and composed configuration belong to one snapshot.
const { recipe, password } = db().transaction(() => {
if (!input.frozenRecipeId)
return {
recipe: {
...captureRecipe(input.game, input.mapSlug, input.rconPassword),
native: input.native,
image: input.instance.image
},
password: input.rconPassword
if (!input.frozenRecipeId) {
const recipe = {
...captureRecipe(input.game, input.mapSlug, input.rconPassword),
native: input.native,
image: input.instance.image
};
if (input.activationId) applyActivationFiles(recipe, input.activationId, true);
return { recipe, password: input.rconPassword };
}
const stored = recipeForDeployment(input.frozenRecipeId)?.recipe;
const migration = db()
.prepare(
@@ -85,6 +86,7 @@ parentPort!.on(
)
throw new Error('electric_cherry_bank.cfg is reserved for instance credentials.');
const payload = {
...(input.activationId ? { stageId: input.activationId } : {}),
name: instance.server,
game: recipe.game,
libraryId: instance.library_id,
src/lib/server/provisioning/native.ts
+86 -9
View File
@@ -18,6 +18,7 @@ import { db } from '../db/index.js';
import { unsettledCancelledCommand } from './cancellation.js';
import { config, relDestForKind, serverDir } from '../config.js';
import {
runtimePasswordForServer,
nativeInstance,
nativeRequest,
unplacedNativeCreation,
@@ -55,6 +56,7 @@ import {
availableServerKeyCount
} from '../fleet/server-keys.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
import { applyActivationFiles } from './activation-files.js';
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
import {
beginDeployment,
@@ -916,7 +918,10 @@ function queueUnhealthyRecovery(job: Job, instance: NativeInstance): void {
}
let payload: Record<string, unknown> = { unhealthyRecovery: true };
if (step === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
const beta = activeBetaRemovalBlocker(
job.server,
job.action === 'rebuild' ? job.id : undefined
);
if (beta) {
hold(job, beta);
return;
@@ -952,6 +957,7 @@ function queueUnhealthyRecovery(job: Job, instance: NativeInstance): void {
unhealthyRecovery: true
};
delete payload.safetyOverride;
delete payload.stageId;
}
const operation = queueHostOperationSerialized(
instance.host_id,
@@ -1701,6 +1707,17 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, 'Waiting for current health evidence before completing verification.');
return;
}
// A queued activation from a previous controller release may still say restart.
// Upgrade only intent that has never submitted a host command.
if (
job.action === 'restart' &&
job.state === 'waiting' &&
!job.submitted_at &&
filesFor(job).length
) {
db().prepare("UPDATE server_deployments SET action='rebuild' WHERE id=?").run(job.id);
job.action = 'rebuild';
}
const reason = eligibility(job);
if (reason) {
hold(job, reason);
@@ -1712,13 +1729,58 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
return;
}
if (job.action === 'destroy' || job.action === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
const beta = activeBetaRemovalBlocker(
job.server,
job.action === 'rebuild' ? job.id : undefined
);
if (beta) {
hold(job, beta);
return;
}
}
const files = job.stage_operation_id ? storedFiles(job) : filesFor(job);
if (
job.action === 'rebuild' &&
files.length &&
!host.inventory.capabilities?.includes('instance.rebuild-files')
) {
hold(job, 'Update the host agent to support beta and campaign file rebuilds.');
return;
}
if (job.action === 'rebuild' && !recipeSummaryForDeployment(job.id)) {
const server = getServer(job.server);
if (!server?.map_slug) throw new Error('A saved map is required before rebuilding.');
const recipe: NativeRecipe = {
...captureRecipePlan(server.game as 't5' | 't6', server.map_slug),
image: instance.image,
native: {
hostId: instance.host_id,
libraryId: instance.library_id,
image: instance.image,
bindAddress: instance.bind_address
}
};
const stored = recipeStorage(recipe);
db().transaction(() => {
db()
.prepare(
'INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)'
)
.run(
job.id,
job.server,
instance!.port,
'zombie',
recipe.mapSlug,
recipe.expectedMap
);
db()
.prepare(
'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)'
)
.run(job.id, stored.recipe, stored.recipeSummary);
})();
}
if (files.length && !job.stage_operation_id) {
db().transaction(() => {
const stage = queueHostOperationSerialized(
@@ -1838,7 +1900,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
'instance.inspect'
);
if (!stillActive(job)) return false;
if (job.action === 'rebuild' && health.state === 'unhealthy') {
if (job.action === 'rebuild' && !files.length && health.state === 'unhealthy') {
if (
!getHost(instance.host_id)?.inventory.capabilities?.includes(
unhealthyRecoveryCapability
@@ -1929,7 +1991,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
phase: 'prepare',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' }
});
preparedPassword = config.rcon.password || randomBytes(32).toString('hex');
preparedPassword =
(files.length ? runtimePasswordForServer(job.server) : undefined) ||
config.rcon.password ||
randomBytes(32).toString('hex');
prepared = await prepareNativeLaunch({
game: prior.game,
mapSlug: prior.mapSlug,
@@ -1940,9 +2005,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
bankToken: bankCredential(instance.id),
rebuild: job.action === 'rebuild' && !migration,
safetyOverride: overrideSafety,
unhealthyRecovery: host.inventory.capabilities?.includes(
unhealthyRecoveryCapability
),
unhealthyRecovery:
!files.length &&
host.inventory.capabilities?.includes(unhealthyRecoveryCapability),
activationId: files.length ? job.id : undefined,
frozenRecipeId
});
if (job.action === 'rebuild' && !migration && !(await checkOccupancy())) return;
@@ -1977,7 +2043,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
return;
}
if (job.action === 'destroy' || job.action === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
const beta = activeBetaRemovalBlocker(
job.server,
job.action === 'rebuild' ? job.id : undefined
);
if (beta) {
hold(job, beta);
return;
@@ -2032,6 +2101,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
prepared.recipe.mapSlug,
preparedPassword
);
if (files.length) applyActivationFiles(expected, job.id);
if (
getServerKey(job.server) !== preparedKey ||
!sameLaunchFiles(prepared.recipe, expected)
@@ -2119,7 +2189,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
recipe.image,
instance!.port
);
for (const file of recipe.files)
for (const file of recipe.files.filter(
(file) =>
!files.some(
(change) =>
change.path ===
posix.join(relDestForKind(file.kind, recipe.game), file.path)
)
))
recordScript(
job.server,
file.path,
src/lib/server/provisioning/requests.ts
+25 -2
View File
@@ -43,7 +43,7 @@ const sourceTables = {
maintenance: { table: 'maintenance_jobs', key: 'id' }
} as const;
/** Submit once per caller attempt; a lost response or terminal failure never replays a restart. */
/** File activations rebuild; maintenance and beta recovery cycles without files restart. */
export function requestRestart(server: string, source: RestartSource): LifecycleOperation {
return db().transaction(() => {
const { table, key } = sourceTables[source.kind];
@@ -54,6 +54,13 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
.get(source.id, server) as
{ provisioning_id: string | null; restarted_at: string | null } | undefined;
if (!row) throw new Error('Restart source no longer exists');
const activatesFiles =
source.kind !== 'maintenance' &&
!!db()
.prepare(
"SELECT 1 FROM staged_file_changes WHERE source_kind=? AND source_id=? AND server=? AND state='pending' LIMIT 1"
)
.get(source.kind, source.id, server);
if (row.provisioning_id) {
const operation = provisioningOperation(row.provisioning_id);
if (!operation)
@@ -62,6 +69,18 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
adoptSubmittedRestart(operation.id, server, row.restarted_at);
return provisioningOperation(operation.id)!;
}
if (
activatesFiles &&
!row.restarted_at &&
!operation.submitted_at &&
operation.state === 'waiting' &&
operation.action === 'restart'
) {
db()
.prepare("UPDATE server_deployments SET action='rebuild' WHERE id=?")
.run(operation.id);
return provisioningOperation(operation.id)!;
}
return operation;
}
let id: string | undefined;
@@ -84,7 +103,11 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
.run(row.restarted_at, id);
}
if (!id) {
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
id = beginDeployment(
server,
activatesFiles && !row.restarted_at ? 'rebuild' : 'restart',
`${source.kind}:${source.id}`
);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty', {
phase: 'eligibility',
src/routes/betas/[id]/+page.svelte
+7 -7
View File
@@ -297,7 +297,7 @@
const restarts =
test?.plan.rollbackRestart === 'manual'
? 'Resume the rollback campaign when ready; files stay unchanged until safety checks pass.'
: 'Files are restored just before restarting, after safety checks pass.';
: 'Servers rebuild with the original files after safety checks pass.';
if (
!(await confirmAction({
title: reject ? 'Reject beta?' : 'Roll back beta?',
@@ -557,7 +557,7 @@
><td colspan="4">
<p class="muted">
Files {server.deployed_at ? 'installed' : 'pending'} ·
Restart {server.restarted_at
Activation {server.restarted_at
? 'submitted'
: 'pending'}{#if beta.wait_for_health}
· Health {server.healthy_at
@@ -734,10 +734,10 @@
</p>
<p>
Confirming creates a campaign and queues the file changes. Each server waits until empty
and restart capacity is available, then captures rollback snapshots, installs its files,
and restarts. Tested servers keep their pre-beta snapshots and only need another
activation if their configuration changed or earlier queued work could replace the
tested files. Eligibility follows each file's current scope.
and rebuild capacity is available, then captures rollback snapshots and rebuilds with
its files. Tested servers keep their pre-beta snapshots and only need another activation
if their configuration changed or earlier queued work could replace the tested files.
Eligibility follows each file's current scope.
</p>
<p class="muted">
Reviewed {$timeFormat.dateTime(preview.createdAt)}. If targets, versions, or composed
@@ -756,7 +756,7 @@
<ServerIdentity {name} /><span
>{preview.servers.find((server) => server.name === name)?.tested
? 'Already tested'
: 'Additional target · restart when empty'}</span
: 'Additional target · rebuild when empty'}</span
>
</li>{/each}
</ul>
src/routes/betas/new/+page.svelte
+3 -3
View File
@@ -989,12 +989,12 @@
? 'Failed checks pause testing.'
: 'Failed checks hold the beta for review.'}
{testPlan.rollbackRestart === 'when-empty'
? 'Rollback restarts wait for empty servers.'
: 'Rollback restores files; restart servers manually.'}
? 'Rollback rebuilds wait for empty servers.'
: 'Rollback is held until you resume its campaign.'}
</p>
<p>
Starting installs each selected version only on matching test servers and
restarts them when empty. Waiting, pausing, and retrying use the original
rebuilds them when empty. Waiting, pausing, and retrying use the original
testing window. Promotion is a separate reviewed action after testing.
</p>
</section>{/if}
src/routes/campaigns/[id]/+page.svelte
+2 -2
View File
@@ -349,7 +349,7 @@
<p>
{detail.watchdogIncidentId
? 'Maintenance rebuilds instance files with the latest promoted setup after checking that the server is safe to update.'
: 'Changes wait for an empty server and restart capacity. Rollback snapshots are captured before applying files, then servers restart and verify health. Tested servers retain their pre-beta snapshots.'}
: 'Changes wait for an empty server and rebuild capacity. Rollback snapshots are captured before servers rebuild with the requested files and verify health. Tested servers retain their pre-beta snapshots.'}
</p>
</div>
</details>
@@ -495,7 +495,7 @@
<dd><RelativeTime value={s.checked_at} /></dd>
</div>{/if}
{#if s.restarted_at}<div>
<dt>Restart requested</dt>
<dt>Rebuild requested</dt>
<dd><RelativeTime value={s.restarted_at} /></dd>
</div>{/if}
{#if s.server_deleted_at}<div>