josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild beta and campaign file activations

Browse Source
This commit is contained in:
josh
2026-10-12 22:52:58 -04:00
parent fa0baa8db5
commit 537b9841a7
22 changed files with 533 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/servers.ts
+5 -2
View File
@@ -190,14 +190,17 @@ export function listServers(): ServerRow[] {
}
/** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */
export function activeBetaRemovalBlocker(server: string): string | null {
export function activeBetaRemovalBlocker(server: string, activationId?: string): string | null {
const betas = db()
.prepare(
`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id=?
AND s.kind='beta' AND s.source_id=CAST(b.id AS TEXT) AND s.server=bs.server
AND bs.provisioning_id=s.operation_id)
ORDER BY b.id`
)
.all(server) as { id: number }[];
.all(server, activationId ?? null) as { id: number }[];
if (!betas.length) return null;
return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map((beta) => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`;
}
src/lib/server/pipeline/beta-plan-runner.ts
+2 -2
View File
@@ -56,11 +56,11 @@ export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void
if (!result.deploymentId)
activity(id, 'Cancelled queued changes; no deployed files need restoration');
else if (plan.rollbackRestart === 'when-empty')
activity(id, 'Queued file restoration and rolling restarts; waiting for empty servers');
activity(id, 'Queued file restoration and rolling rebuilds; waiting for empty servers');
else
activity(
id,
'File restoration is held; resume the rollback campaign to apply it with a safe restart'
'File restoration is held; resume the rollback campaign to apply it with a safe rebuild'
);
if (result.warnings.length) activity(id, result.warnings.join('; '));
}
src/lib/server/provisioning/activation-files.ts
+44
View File
@@ -0,0 +1,44 @@
import { db } from '../db/index.js';
import { blobSize, getFileBlob } from '../db/file-blobs.js';
import type { LaunchRecipePlan } from '../fleet/launch-recipes.js';
import type { StagedFileChange } from '../fleet/staged-files.js';
/** Build the replacement with the exact requested additions, updates and removals. */
export function applyActivationFiles(
recipe: LaunchRecipePlan,
operationId: string,
includeContent = false
): void {
const changes = db()
.prepare(
`SELECT f.*, ${blobSize('f.content_hash')} AS size, v.version FROM staged_file_changes f
JOIN provisioning_sources s ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
LEFT JOIN script_versions v ON v.id=f.script_version_id
WHERE s.operation_id=? AND f.state='pending' ORDER BY f.id`
)
.all(operationId) as (StagedFileChange & {
content_hash: string | null;
size: number;
version: number | null;
})[];
for (const change of changes) {
if (change.kind !== 'script' && change.kind !== 'config' && change.kind !== 'plugin')
throw new Error('File activation requires a runtime file kind.');
recipe.files = recipe.files.filter(
(file) => file.kind !== change.kind || file.path !== change.rel_path
);
if (!change.content_hash) continue;
recipe.files.push({
kind: change.kind,
name: change.rel_path,
path: change.rel_path,
versionId: change.script_version_id,
version: change.version,
sha256: change.content_hash,
size: change.size,
...(includeContent
? { content: getFileBlob(change.content_hash)!.toString('base64') }
: {})
});
}
}
src/lib/server/provisioning/launch-preparation.ts
+2
View File
@@ -19,6 +19,8 @@ export interface NativeLaunchInput {
unhealthyRecovery?: boolean;
/** Rebalance replays recorded deployed bytes instead of current promoted files. */
frozenRecipeId?: string;
/** Pending beta/campaign files override promoted launch files. */
activationId?: string;
}
export interface PreparedNativeLaunch {
src/lib/server/provisioning/launch-worker.ts
+10 -8
View File
@@ -1,5 +1,6 @@
import { parentPort } from 'node:worker_threads';
import { posix } from 'node:path';
import { applyActivationFiles } from './activation-files.js';
import { db } from '../db/index.js';
import { relDestForKind } from '../config.js';
import {
@@ -32,15 +33,15 @@ parentPort!.on(
const input = message.input;
// All file versions and composed configuration belong to one snapshot.
const { recipe, password } = db().transaction(() => {
if (!input.frozenRecipeId)
return {
recipe: {
...captureRecipe(input.game, input.mapSlug, input.rconPassword),
native: input.native,
image: input.instance.image
},
password: input.rconPassword
if (!input.frozenRecipeId) {
const recipe = {
...captureRecipe(input.game, input.mapSlug, input.rconPassword),
native: input.native,
image: input.instance.image
};
if (input.activationId) applyActivationFiles(recipe, input.activationId, true);
return { recipe, password: input.rconPassword };
}
const stored = recipeForDeployment(input.frozenRecipeId)?.recipe;
const migration = db()
.prepare(
@@ -85,6 +86,7 @@ parentPort!.on(
)
throw new Error('electric_cherry_bank.cfg is reserved for instance credentials.');
const payload = {
...(input.activationId ? { stageId: input.activationId } : {}),
name: instance.server,
game: recipe.game,
libraryId: instance.library_id,
src/lib/server/provisioning/native.ts
+86 -9
View File
@@ -18,6 +18,7 @@ import { db } from '../db/index.js';
import { unsettledCancelledCommand } from './cancellation.js';
import { config, relDestForKind, serverDir } from '../config.js';
import {
runtimePasswordForServer,
nativeInstance,
nativeRequest,
unplacedNativeCreation,
@@ -55,6 +56,7 @@ import {
availableServerKeyCount
} from '../fleet/server-keys.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
import { applyActivationFiles } from './activation-files.js';
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
import {
beginDeployment,
@@ -916,7 +918,10 @@ function queueUnhealthyRecovery(job: Job, instance: NativeInstance): void {
}
let payload: Record<string, unknown> = { unhealthyRecovery: true };
if (step === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
const beta = activeBetaRemovalBlocker(
job.server,
job.action === 'rebuild' ? job.id : undefined
);
if (beta) {
hold(job, beta);
return;
@@ -952,6 +957,7 @@ function queueUnhealthyRecovery(job: Job, instance: NativeInstance): void {
unhealthyRecovery: true
};
delete payload.safetyOverride;
delete payload.stageId;
}
const operation = queueHostOperationSerialized(
instance.host_id,
@@ -1701,6 +1707,17 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, 'Waiting for current health evidence before completing verification.');
return;
}
// A queued activation from a previous controller release may still say restart.
// Upgrade only intent that has never submitted a host command.
if (
job.action === 'restart' &&
job.state === 'waiting' &&
!job.submitted_at &&
filesFor(job).length
) {
db().prepare("UPDATE server_deployments SET action='rebuild' WHERE id=?").run(job.id);
job.action = 'rebuild';
}
const reason = eligibility(job);
if (reason) {
hold(job, reason);
@@ -1712,13 +1729,58 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
return;
}
if (job.action === 'destroy' || job.action === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
const beta = activeBetaRemovalBlocker(
job.server,
job.action === 'rebuild' ? job.id : undefined
);
if (beta) {
hold(job, beta);
return;
}
}
const files = job.stage_operation_id ? storedFiles(job) : filesFor(job);
if (
job.action === 'rebuild' &&
files.length &&
!host.inventory.capabilities?.includes('instance.rebuild-files')
) {
hold(job, 'Update the host agent to support beta and campaign file rebuilds.');
return;
}
if (job.action === 'rebuild' && !recipeSummaryForDeployment(job.id)) {
const server = getServer(job.server);
if (!server?.map_slug) throw new Error('A saved map is required before rebuilding.');
const recipe: NativeRecipe = {
...captureRecipePlan(server.game as 't5' | 't6', server.map_slug),
image: instance.image,
native: {
hostId: instance.host_id,
libraryId: instance.library_id,
image: instance.image,
bindAddress: instance.bind_address
}
};
const stored = recipeStorage(recipe);
db().transaction(() => {
db()
.prepare(
'INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)'
)
.run(
job.id,
job.server,
instance!.port,
'zombie',
recipe.mapSlug,
recipe.expectedMap
);
db()
.prepare(
'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)'
)
.run(job.id, stored.recipe, stored.recipeSummary);
})();
}
if (files.length && !job.stage_operation_id) {
db().transaction(() => {
const stage = queueHostOperationSerialized(
@@ -1838,7 +1900,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
'instance.inspect'
);
if (!stillActive(job)) return false;
if (job.action === 'rebuild' && health.state === 'unhealthy') {
if (job.action === 'rebuild' && !files.length && health.state === 'unhealthy') {
if (
!getHost(instance.host_id)?.inventory.capabilities?.includes(
unhealthyRecoveryCapability
@@ -1929,7 +1991,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
phase: 'prepare',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' }
});
preparedPassword = config.rcon.password || randomBytes(32).toString('hex');
preparedPassword =
(files.length ? runtimePasswordForServer(job.server) : undefined) ||
config.rcon.password ||
randomBytes(32).toString('hex');
prepared = await prepareNativeLaunch({
game: prior.game,
mapSlug: prior.mapSlug,
@@ -1940,9 +2005,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
bankToken: bankCredential(instance.id),
rebuild: job.action === 'rebuild' && !migration,
safetyOverride: overrideSafety,
unhealthyRecovery: host.inventory.capabilities?.includes(
unhealthyRecoveryCapability
),
unhealthyRecovery:
!files.length &&
host.inventory.capabilities?.includes(unhealthyRecoveryCapability),
activationId: files.length ? job.id : undefined,
frozenRecipeId
});
if (job.action === 'rebuild' && !migration && !(await checkOccupancy())) return;
@@ -1977,7 +2043,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
return;
}
if (job.action === 'destroy' || job.action === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
const beta = activeBetaRemovalBlocker(
job.server,
job.action === 'rebuild' ? job.id : undefined
);
if (beta) {
hold(job, beta);
return;
@@ -2032,6 +2101,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
prepared.recipe.mapSlug,
preparedPassword
);
if (files.length) applyActivationFiles(expected, job.id);
if (
getServerKey(job.server) !== preparedKey ||
!sameLaunchFiles(prepared.recipe, expected)
@@ -2119,7 +2189,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
recipe.image,
instance!.port
);
for (const file of recipe.files)
for (const file of recipe.files.filter(
(file) =>
!files.some(
(change) =>
change.path ===
posix.join(relDestForKind(file.kind, recipe.game), file.path)
)
))
recordScript(
job.server,
file.path,
src/lib/server/provisioning/requests.ts
+25 -2
View File
@@ -43,7 +43,7 @@ const sourceTables = {
maintenance: { table: 'maintenance_jobs', key: 'id' }
} as const;
/** Submit once per caller attempt; a lost response or terminal failure never replays a restart. */
/** File activations rebuild; maintenance and beta recovery cycles without files restart. */
export function requestRestart(server: string, source: RestartSource): LifecycleOperation {
return db().transaction(() => {
const { table, key } = sourceTables[source.kind];
@@ -54,6 +54,13 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
.get(source.id, server) as
{ provisioning_id: string | null; restarted_at: string | null } | undefined;
if (!row) throw new Error('Restart source no longer exists');
const activatesFiles =
source.kind !== 'maintenance' &&
!!db()
.prepare(
"SELECT 1 FROM staged_file_changes WHERE source_kind=? AND source_id=? AND server=? AND state='pending' LIMIT 1"
)
.get(source.kind, source.id, server);
if (row.provisioning_id) {
const operation = provisioningOperation(row.provisioning_id);
if (!operation)
@@ -62,6 +69,18 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
adoptSubmittedRestart(operation.id, server, row.restarted_at);
return provisioningOperation(operation.id)!;
}
if (
activatesFiles &&
!row.restarted_at &&
!operation.submitted_at &&
operation.state === 'waiting' &&
operation.action === 'restart'
) {
db()
.prepare("UPDATE server_deployments SET action='rebuild' WHERE id=?")
.run(operation.id);
return provisioningOperation(operation.id)!;
}
return operation;
}
let id: string | undefined;
@@ -84,7 +103,11 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
.run(row.restarted_at, id);
}
if (!id) {
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
id = beginDeployment(
server,
activatesFiles && !row.restarted_at ? 'rebuild' : 'restart',
`${source.kind}:${source.id}`
);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty', {
phase: 'eligibility',