4a318c36ad
CI / build-and-push (push) Successful in 40s
Campus level: "Fill All DCs" instantly fills all operational DCs with selected SKU in one click. "Retrofit Campus" queues a staggered retrofit with configurable concurrency (1/10%/25%/custom) so only a fraction of DCs go offline at a time, preserving capacity during the upgrade. Cluster level: "Fill All DCs" fills across all campuses in one action. The game engine automatically advances the retrofit queue each tick, promoting pending DCs as active ones complete. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
532 lines
19 KiB
TypeScript
532 lines
19 KiB
TypeScript
import type {
|
|
GameState, InfrastructureState, Cluster, Campus, DataCenter,
|
|
DeploymentCohort, NetworkHealthState, PipelineStage, RackSkuId,
|
|
CampusRetrofitQueue,
|
|
} from '@ai-tycoon/shared';
|
|
import {
|
|
LOCATION_CONFIGS,
|
|
RACK_SKU_CONFIGS,
|
|
DC_TIER_CONFIGS,
|
|
BASE_ENERGY_COST_PER_FLOP,
|
|
BASE_MAINTENANCE_PER_RACK,
|
|
COOLING_FAILURE_REDUCTION,
|
|
REDUNDANCY_FAILURE_REDUCTION,
|
|
RACK_REPAIR_BASE_TICKS,
|
|
NETWORK_TOPOLOGY,
|
|
COHORT_SCALE_FACTOR,
|
|
PIPELINE_ORDER_BASE_TICKS,
|
|
networkSlotsRequired,
|
|
} from '@ai-tycoon/shared';
|
|
import type { TickNotification } from '../tick';
|
|
|
|
export interface InfraTickResult {
|
|
infrastructure: InfrastructureState;
|
|
notifications: TickNotification[];
|
|
repairCosts: number;
|
|
}
|
|
|
|
const PIPELINE_ADVANCE_ORDER: PipelineStage[] = [
|
|
'ordered', 'manufacturing', 'receiving', 'installation', 'testing',
|
|
];
|
|
|
|
function nextStage(stage: PipelineStage): PipelineStage | 'production' {
|
|
const idx = PIPELINE_ADVANCE_ORDER.indexOf(stage);
|
|
if (idx === -1 || idx === PIPELINE_ADVANCE_ORDER.length - 1) return 'production';
|
|
return PIPELINE_ADVANCE_ORDER[idx + 1];
|
|
}
|
|
|
|
function cohortStageTotal(stage: PipelineStage, skuId: string, count: number): number {
|
|
const sku = RACK_SKU_CONFIGS[skuId as keyof typeof RACK_SKU_CONFIGS];
|
|
const timings = sku.pipelineTimeTicks;
|
|
let base: number;
|
|
switch (stage) {
|
|
case 'ordered': base = PIPELINE_ORDER_BASE_TICKS; break;
|
|
case 'manufacturing': base = timings.manufacturing; break;
|
|
case 'receiving': base = timings.receiving; break;
|
|
case 'installation': base = timings.installation; break;
|
|
case 'testing': base = timings.testing; break;
|
|
case 'repair': base = RACK_REPAIR_BASE_TICKS; break;
|
|
case 'decommission': base = timings.installation; break;
|
|
default: base = 0;
|
|
}
|
|
return Math.ceil(base * (1 + COHORT_SCALE_FACTOR * count));
|
|
}
|
|
|
|
function stageSpeed(stage: PipelineStage, engEff: number, opsEff: number): number {
|
|
switch (stage) {
|
|
case 'manufacturing': return 1 + engEff * 0.1;
|
|
case 'installation':
|
|
case 'testing':
|
|
case 'decommission': return 1 + opsEff * 0.1;
|
|
case 'repair': return 1 + opsEff * 0.05;
|
|
default: return 1;
|
|
}
|
|
}
|
|
|
|
function binomialSample(n: number, p: number): number {
|
|
if (n <= 0 || p <= 0) return 0;
|
|
if (p >= 1) return n;
|
|
const expected = n * p;
|
|
const base = Math.floor(expected);
|
|
const frac = expected - base;
|
|
return base + (Math.random() < frac ? 1 : 0);
|
|
}
|
|
|
|
function computeNetworkHealth(computeRacksOnline: number): NetworkHealthState {
|
|
if (computeRacksOnline <= 0) {
|
|
return { tier1Required: 0, tier1Healthy: 0, tier2Required: 0, tier2Healthy: 0, tier3Required: 0, tier3Healthy: 0, racksDisconnected: 0 };
|
|
}
|
|
const tier1 = Math.ceil(computeRacksOnline / NETWORK_TOPOLOGY.tier1PerCompute);
|
|
const tier2 = Math.ceil(tier1 / NETWORK_TOPOLOGY.tier2PerTier1);
|
|
const tier3 = NETWORK_TOPOLOGY.tier3PerDC;
|
|
return {
|
|
tier1Required: tier1,
|
|
tier1Healthy: tier1,
|
|
tier2Required: tier2,
|
|
tier2Healthy: tier2,
|
|
tier3Required: tier3,
|
|
tier3Healthy: tier3,
|
|
racksDisconnected: 0,
|
|
};
|
|
}
|
|
|
|
function processNetworkFailures(
|
|
nh: NetworkHealthState,
|
|
computeRacksOnline: number,
|
|
networkResearchBonus: number,
|
|
): { networkHealth: NetworkHealthState; racksDisconnected: number } {
|
|
if (computeRacksOnline <= 0) {
|
|
return { networkHealth: nh, racksDisconnected: 0 };
|
|
}
|
|
|
|
let racksDisconnected = 0;
|
|
|
|
const t1Rate = NETWORK_TOPOLOGY.tier1FailureRate * (1 - networkResearchBonus);
|
|
const t1Failures = binomialSample(nh.tier1Required, t1Rate);
|
|
const tier1Healthy = nh.tier1Required - t1Failures;
|
|
racksDisconnected += t1Failures * NETWORK_TOPOLOGY.tier1BlastRadius;
|
|
|
|
const t2Rate = NETWORK_TOPOLOGY.tier2FailureRate * (1 - networkResearchBonus);
|
|
const t2Failures = binomialSample(nh.tier2Required, t2Rate);
|
|
const tier2Healthy = nh.tier2Required - t2Failures;
|
|
racksDisconnected += t2Failures * NETWORK_TOPOLOGY.tier1BlastRadius * NETWORK_TOPOLOGY.tier2BlastRadiusMultiplier;
|
|
|
|
const t3Rate = NETWORK_TOPOLOGY.tier3FailureRate * (1 - networkResearchBonus);
|
|
const t3Failures = binomialSample(nh.tier3Required, t3Rate);
|
|
const tier3Healthy = nh.tier3Required - t3Failures;
|
|
if (t3Failures > 0) {
|
|
racksDisconnected = computeRacksOnline;
|
|
}
|
|
|
|
racksDisconnected = Math.min(racksDisconnected, computeRacksOnline);
|
|
|
|
return {
|
|
networkHealth: {
|
|
...nh,
|
|
tier1Healthy,
|
|
tier2Healthy,
|
|
tier3Healthy,
|
|
racksDisconnected,
|
|
},
|
|
racksDisconnected,
|
|
};
|
|
}
|
|
|
|
export function processInfrastructure(state: GameState): InfraTickResult {
|
|
const notifications: TickNotification[] = [];
|
|
let repairCosts = 0;
|
|
|
|
const engEff = state.talent.departments.engineering.effectiveness;
|
|
const opsEff = state.talent.departments.operations.effectiveness;
|
|
const qaResearchBonus = state.research.completedResearch.includes('quality-assurance') ? 0.25 : 0;
|
|
const netResearch1 = state.research.completedResearch.includes('network-engineering-i') ? 0.4 : 0;
|
|
const netResearch2 = state.research.completedResearch.includes('network-engineering-ii') ? 0.5 : 0;
|
|
const networkResearchBonus = Math.min(0.8, netResearch1 + netResearch2);
|
|
|
|
let totalFlops = 0;
|
|
let totalUptime = 0;
|
|
let totalRackCount = 0;
|
|
let totalComputeRackCount = 0;
|
|
let totalDataCenterCount = 0;
|
|
let dcWithRacks = 0;
|
|
|
|
const clusters: Cluster[] = state.infrastructure.clusters.map(cluster => {
|
|
// Advance cluster construction
|
|
if (cluster.status === 'constructing') {
|
|
const newProgress = cluster.constructionProgress + 1;
|
|
if (newProgress >= cluster.constructionTotal) {
|
|
notifications.push({
|
|
title: 'Cluster Online',
|
|
message: `${cluster.name} cluster in ${LOCATION_CONFIGS[cluster.locationId].name} is now operational!`,
|
|
type: 'success',
|
|
});
|
|
return { ...cluster, constructionProgress: cluster.constructionTotal, status: 'operational' as const, campuses: cluster.campuses };
|
|
}
|
|
return { ...cluster, constructionProgress: newProgress };
|
|
}
|
|
|
|
const campuses: Campus[] = cluster.campuses.map(campus => {
|
|
// Advance campus construction
|
|
if (campus.status === 'constructing') {
|
|
const newProgress = campus.constructionProgress + 1;
|
|
if (newProgress >= campus.constructionTotal) {
|
|
notifications.push({
|
|
title: 'Campus Ready',
|
|
message: `Campus ${campus.name} is now operational!`,
|
|
type: 'success',
|
|
});
|
|
return { ...campus, constructionProgress: campus.constructionTotal, status: 'operational' as const, dataCenters: campus.dataCenters };
|
|
}
|
|
return { ...campus, constructionProgress: newProgress };
|
|
}
|
|
|
|
const dataCenters: DataCenter[] = campus.dataCenters.map(dc => {
|
|
// Advance DC construction
|
|
if (dc.status === 'constructing') {
|
|
const newProgress = dc.constructionProgress + 1;
|
|
if (newProgress >= dc.constructionTotal) {
|
|
notifications.push({
|
|
title: 'Data Center Online',
|
|
message: `${dc.name} is now operational!`,
|
|
type: 'success',
|
|
});
|
|
return { ...dc, constructionProgress: dc.constructionTotal, status: 'operational' as const };
|
|
}
|
|
return { ...dc, constructionProgress: newProgress };
|
|
}
|
|
|
|
let computeRacksOnline = dc.computeRacksOnline;
|
|
let dcRepairCosts = 0;
|
|
|
|
// Process retrofit
|
|
if (dc.status === 'retrofitting' && dc.retrofitState) {
|
|
const rs = { ...dc.retrofitState };
|
|
rs.progress += (1 + opsEff * 0.1);
|
|
|
|
if (rs.progress >= rs.total) {
|
|
if (rs.phase === 'decommissioning') {
|
|
const installSku = RACK_SKU_CONFIGS[rs.toSkuId];
|
|
const installTotal = cohortStageTotal('installation', rs.toSkuId, rs.racksRemaining);
|
|
return {
|
|
...dc,
|
|
computeRacksOnline: 0,
|
|
computeRacksFailed: 0,
|
|
rackSkuId: rs.toSkuId,
|
|
deploymentCohorts: [{
|
|
id: `retrofit-${dc.id}-${Date.now()}`,
|
|
count: rs.racksRemaining,
|
|
skuId: rs.toSkuId,
|
|
stage: 'installation' as PipelineStage,
|
|
stageProgress: 0,
|
|
stageTotal: installTotal,
|
|
repairCount: 0,
|
|
}],
|
|
retrofitState: {
|
|
...rs,
|
|
phase: 'installing' as const,
|
|
progress: 0,
|
|
total: installTotal,
|
|
},
|
|
networkHealth: computeNetworkHealth(0),
|
|
effectiveComputeRacks: 0,
|
|
usedSlots: 0,
|
|
usedPowerKW: 0,
|
|
currentUptime: 0,
|
|
energyCostPerTick: DC_TIER_CONFIGS[dc.tier].baseEnergyCostPerTick * LOCATION_CONFIGS[cluster.locationId].energyCostMultiplier,
|
|
maintenanceCostPerTick: 0,
|
|
};
|
|
} else {
|
|
notifications.push({
|
|
title: 'Retrofit Complete',
|
|
message: `${dc.name} retrofit to ${RACK_SKU_CONFIGS[rs.toSkuId].name} is complete!`,
|
|
type: 'success',
|
|
});
|
|
return {
|
|
...dc,
|
|
status: 'operational' as const,
|
|
retrofitState: null,
|
|
};
|
|
}
|
|
}
|
|
return { ...dc, retrofitState: rs };
|
|
}
|
|
|
|
// Process deployment cohorts
|
|
const updatedCohorts: DeploymentCohort[] = [];
|
|
let racksJustOnlined = 0;
|
|
let racksFailedTesting = 0;
|
|
|
|
for (const cohort of dc.deploymentCohorts) {
|
|
const speed = stageSpeed(cohort.stage, engEff, opsEff);
|
|
const newProgress = cohort.stageProgress + speed;
|
|
|
|
if (newProgress < cohort.stageTotal) {
|
|
updatedCohorts.push({ ...cohort, stageProgress: newProgress });
|
|
continue;
|
|
}
|
|
|
|
if (cohort.stage === 'decommission') {
|
|
continue;
|
|
}
|
|
|
|
if (cohort.stage === 'repair') {
|
|
const testTotal = cohortStageTotal('testing', cohort.skuId, cohort.count);
|
|
updatedCohorts.push({
|
|
...cohort,
|
|
stage: 'testing',
|
|
stageProgress: 0,
|
|
stageTotal: testTotal,
|
|
});
|
|
continue;
|
|
}
|
|
|
|
const next = nextStage(cohort.stage);
|
|
|
|
if (next === 'production') {
|
|
const sku = RACK_SKU_CONFIGS[cohort.skuId];
|
|
const effectiveFailRate = sku.testFailureRate
|
|
* (1 - dc.coolingLevel * COOLING_FAILURE_REDUCTION)
|
|
* (1 - opsEff * 0.2)
|
|
* (1 - qaResearchBonus);
|
|
|
|
const failed = binomialSample(cohort.count, effectiveFailRate);
|
|
const passed = cohort.count - failed;
|
|
|
|
racksJustOnlined += passed;
|
|
|
|
if (failed > 0) {
|
|
racksFailedTesting += failed;
|
|
const repairCost = sku.baseCost * sku.repairCostFraction * failed;
|
|
dcRepairCosts += repairCost;
|
|
|
|
updatedCohorts.push({
|
|
id: `repair-${cohort.id}`,
|
|
count: failed,
|
|
skuId: cohort.skuId,
|
|
stage: 'repair',
|
|
stageProgress: 0,
|
|
stageTotal: cohortStageTotal('repair', cohort.skuId, failed),
|
|
repairCount: cohort.repairCount + 1,
|
|
});
|
|
}
|
|
} else {
|
|
const total = cohortStageTotal(next, cohort.skuId, cohort.count);
|
|
updatedCohorts.push({
|
|
...cohort,
|
|
stage: next,
|
|
stageProgress: 0,
|
|
stageTotal: total,
|
|
});
|
|
}
|
|
}
|
|
|
|
computeRacksOnline += racksJustOnlined;
|
|
|
|
if (racksFailedTesting > 0) {
|
|
const skuName = dc.rackSkuId ? RACK_SKU_CONFIGS[dc.rackSkuId].name : 'Unknown';
|
|
notifications.push({
|
|
title: 'Racks Failed Testing',
|
|
message: `${dc.name}: ${racksFailedTesting} ${skuName} rack${racksFailedTesting > 1 ? 's' : ''} failed QA — repair batch created.`,
|
|
type: 'warning',
|
|
});
|
|
}
|
|
|
|
if (racksJustOnlined > 0 && updatedCohorts.filter(c => c.stage !== 'repair').length === 0) {
|
|
notifications.push({
|
|
title: 'Deployment Complete',
|
|
message: `${dc.name}: all racks deployed and online!`,
|
|
type: 'success',
|
|
});
|
|
}
|
|
|
|
// Production failures (statistical)
|
|
if (computeRacksOnline > 0 && dc.rackSkuId) {
|
|
const sku = RACK_SKU_CONFIGS[dc.rackSkuId];
|
|
const effectiveRate = sku.productionFailureRate
|
|
* (1 - dc.coolingLevel * COOLING_FAILURE_REDUCTION)
|
|
* (1 - dc.redundancyLevel * REDUNDANCY_FAILURE_REDUCTION);
|
|
|
|
const prodFailures = binomialSample(computeRacksOnline, effectiveRate);
|
|
if (prodFailures > 0) {
|
|
computeRacksOnline -= prodFailures;
|
|
const repairCost = sku.baseCost * sku.repairCostFraction * prodFailures;
|
|
dcRepairCosts += repairCost;
|
|
|
|
updatedCohorts.push({
|
|
id: `prodfail-${dc.id}-${Date.now()}`,
|
|
count: prodFailures,
|
|
skuId: dc.rackSkuId,
|
|
stage: 'repair',
|
|
stageProgress: 0,
|
|
stageTotal: cohortStageTotal('repair', dc.rackSkuId, prodFailures),
|
|
repairCount: 0,
|
|
});
|
|
|
|
|
|
}
|
|
}
|
|
|
|
repairCosts += dcRepairCosts;
|
|
|
|
// Network health
|
|
const baseNetworkHealth = computeNetworkHealth(computeRacksOnline);
|
|
const { networkHealth, racksDisconnected } = processNetworkFailures(
|
|
baseNetworkHealth, computeRacksOnline, networkResearchBonus,
|
|
);
|
|
|
|
if (racksDisconnected > 0) {
|
|
if (networkHealth.tier3Healthy < networkHealth.tier3Required) {
|
|
notifications.push({
|
|
title: 'Core Network Failure',
|
|
message: `${dc.name}: Tier-3 core switch failure — entire DC disconnected!`,
|
|
type: 'danger',
|
|
});
|
|
} else if (racksDisconnected >= NETWORK_TOPOLOGY.tier1BlastRadius * NETWORK_TOPOLOGY.tier2BlastRadiusMultiplier) {
|
|
notifications.push({
|
|
title: 'Network Switch Failure',
|
|
message: `${dc.name}: Tier-2 aggregation failure — ${racksDisconnected} racks disconnected.`,
|
|
type: 'warning',
|
|
});
|
|
}
|
|
}
|
|
|
|
const effectiveComputeRacks = computeRacksOnline - racksDisconnected;
|
|
|
|
// Compute aggregates for this DC
|
|
const location = LOCATION_CONFIGS[cluster.locationId];
|
|
const tierConfig = DC_TIER_CONFIGS[dc.tier];
|
|
const pipelineRacks = updatedCohorts
|
|
.filter(c => c.stage !== 'decommission')
|
|
.reduce((sum, c) => sum + c.count, 0);
|
|
const computeRacksFailed = updatedCohorts
|
|
.filter(c => c.stage === 'repair')
|
|
.reduce((sum, c) => sum + c.count, 0);
|
|
const totalRacksInDc = computeRacksOnline + pipelineRacks;
|
|
const netSlots = networkSlotsRequired(computeRacksOnline + pipelineRacks);
|
|
const usedSlots = computeRacksOnline + pipelineRacks + netSlots;
|
|
|
|
let usedPowerKW = 0;
|
|
let dcFlops = 0;
|
|
if (dc.rackSkuId && computeRacksOnline > 0) {
|
|
const sku = RACK_SKU_CONFIGS[dc.rackSkuId];
|
|
usedPowerKW = computeRacksOnline * sku.powerDrawKW;
|
|
dcFlops = effectiveComputeRacks * sku.flopsPerRack;
|
|
}
|
|
|
|
const energyCostPerTick = (tierConfig.baseEnergyCostPerTick + usedPowerKW * BASE_ENERGY_COST_PER_FLOP)
|
|
* location.energyCostMultiplier;
|
|
const maintenanceCostPerTick = totalRacksInDc * BASE_MAINTENANCE_PER_RACK;
|
|
|
|
const currentUptime = totalRacksInDc > 0 ? effectiveComputeRacks / totalRacksInDc : 1;
|
|
|
|
totalFlops += dcFlops;
|
|
totalRackCount += totalRacksInDc + netSlots;
|
|
totalComputeRackCount += totalRacksInDc;
|
|
totalDataCenterCount++;
|
|
if (totalRacksInDc > 0) {
|
|
totalUptime += currentUptime;
|
|
dcWithRacks++;
|
|
}
|
|
|
|
return {
|
|
...dc,
|
|
computeRacksOnline,
|
|
computeRacksFailed,
|
|
deploymentCohorts: updatedCohorts,
|
|
networkHealth,
|
|
effectiveComputeRacks,
|
|
usedSlots,
|
|
usedPowerKW,
|
|
energyCostPerTick,
|
|
maintenanceCostPerTick,
|
|
currentUptime,
|
|
};
|
|
});
|
|
|
|
// Process campus retrofit queue
|
|
let finalDCs = dataCenters;
|
|
let updatedQueue: CampusRetrofitQueue | null = campus.retrofitQueue ?? null;
|
|
|
|
if (updatedQueue && updatedQueue.pendingDCIds.length + updatedQueue.activeDCIds.length > 0) {
|
|
updatedQueue = { ...updatedQueue };
|
|
|
|
// Detect DCs that just completed retrofit (were active, now operational)
|
|
const newlyCompleted = finalDCs.filter(
|
|
dc => updatedQueue!.activeDCIds.includes(dc.id) && dc.status === 'operational',
|
|
);
|
|
|
|
if (newlyCompleted.length > 0) {
|
|
updatedQueue.activeDCIds = updatedQueue.activeDCIds.filter(
|
|
id => !newlyCompleted.some(dc => dc.id === id),
|
|
);
|
|
updatedQueue.completedDCIds = [
|
|
...updatedQueue.completedDCIds,
|
|
...newlyCompleted.map(dc => dc.id),
|
|
];
|
|
}
|
|
|
|
// Promote DCs from pending to active
|
|
const slotsAvailable = updatedQueue.maxConcurrent - updatedQueue.activeDCIds.length;
|
|
if (slotsAvailable > 0 && updatedQueue.pendingDCIds.length > 0) {
|
|
const toStart = updatedQueue.pendingDCIds.slice(0, slotsAvailable);
|
|
updatedQueue.pendingDCIds = updatedQueue.pendingDCIds.slice(toStart.length);
|
|
updatedQueue.activeDCIds = [...updatedQueue.activeDCIds, ...toStart];
|
|
|
|
finalDCs = finalDCs.map(dc => {
|
|
if (!toStart.includes(dc.id)) return dc;
|
|
if (dc.status !== 'operational' || !dc.rackSkuId) return dc;
|
|
|
|
const pipelineCount = dc.deploymentCohorts.filter(c => c.stage !== 'decommission').reduce((sum, c) => sum + c.count, 0);
|
|
const totalRacks = dc.computeRacksOnline + pipelineCount;
|
|
if (totalRacks <= 0) return dc;
|
|
|
|
const oldSku = RACK_SKU_CONFIGS[dc.rackSkuId as RackSkuId];
|
|
const decommTicks = Math.ceil(oldSku.pipelineTimeTicks.installation * (1 + COHORT_SCALE_FACTOR * totalRacks));
|
|
|
|
return {
|
|
...dc,
|
|
status: 'retrofitting' as const,
|
|
deploymentCohorts: [],
|
|
retrofitState: {
|
|
fromSkuId: dc.rackSkuId as RackSkuId,
|
|
toSkuId: updatedQueue!.targetSkuId,
|
|
phase: 'decommissioning' as const,
|
|
progress: 0,
|
|
total: decommTicks,
|
|
racksRemaining: totalRacks,
|
|
},
|
|
};
|
|
});
|
|
}
|
|
|
|
// Check if queue is complete
|
|
if (updatedQueue.pendingDCIds.length === 0 && updatedQueue.activeDCIds.length === 0) {
|
|
notifications.push({
|
|
title: 'Campus Retrofit Complete',
|
|
message: `All DCs in ${campus.name} have been retrofitted to ${RACK_SKU_CONFIGS[updatedQueue.targetSkuId].name}!`,
|
|
type: 'success',
|
|
});
|
|
updatedQueue = null;
|
|
}
|
|
}
|
|
|
|
return { ...campus, dataCenters: finalDCs, retrofitQueue: updatedQueue };
|
|
});
|
|
|
|
return { ...cluster, campuses };
|
|
});
|
|
|
|
return {
|
|
infrastructure: {
|
|
clusters,
|
|
totalFlops,
|
|
totalUptime: dcWithRacks > 0 ? totalUptime / dcWithRacks : 1,
|
|
totalRackCount,
|
|
totalComputeRackCount,
|
|
totalDataCenterCount,
|
|
},
|
|
notifications,
|
|
repairCosts,
|
|
};
|
|
}
|