|
|
|
@@ -1,8 +1,8 @@
|
|
|
|
|
import type {
|
|
|
|
|
GameState, InfrastructureState, Cluster, Campus, DataCenter,
|
|
|
|
|
DeploymentCohort, PipelineStage, RackSkuId, NetworkSwitch,
|
|
|
|
|
DeploymentCohort, PipelineStage, RackSkuId,
|
|
|
|
|
SwitchTier, DCNetworkSummary, CampusNetworkSummary, ClusterNetworkSummary,
|
|
|
|
|
CampusRetrofitQueue, DCTier, IntraNodeInterconnect, NetworkFabric, RackSkuConfig,
|
|
|
|
|
RepairBatch, CampusRetrofitQueue, DCTier, IntraNodeInterconnect, NetworkFabric, RackSkuConfig,
|
|
|
|
|
} from '@ai-tycoon/shared';
|
|
|
|
|
import {
|
|
|
|
|
LOCATION_CONFIGS,
|
|
|
|
@@ -83,357 +83,202 @@ function binomialSample(n: number, p: number): number {
|
|
|
|
|
return base + (Math.random() < frac ? 1 : 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// --- Network Topology Construction ---
|
|
|
|
|
// --- Aggregate Network Model ---
|
|
|
|
|
|
|
|
|
|
let switchIdCounter = 0;
|
|
|
|
|
|
|
|
|
|
function createSwitch(
|
|
|
|
|
tier: SwitchTier,
|
|
|
|
|
dcId: string | null,
|
|
|
|
|
campusId: string | null,
|
|
|
|
|
clusterId: string | null,
|
|
|
|
|
): NetworkSwitch {
|
|
|
|
|
const config = SWITCH_TIER_CONFIGS[tier];
|
|
|
|
|
return {
|
|
|
|
|
id: `${tier}-${dcId ?? campusId ?? clusterId ?? 'x'}-${switchIdCounter++}`,
|
|
|
|
|
tier,
|
|
|
|
|
status: 'healthy',
|
|
|
|
|
dcId, campusId, clusterId,
|
|
|
|
|
uplinkIds: [],
|
|
|
|
|
downlinkIds: [],
|
|
|
|
|
activeUplinks: config.uplinkCount,
|
|
|
|
|
totalUplinks: config.uplinkCount,
|
|
|
|
|
effectiveBandwidth: 1.0,
|
|
|
|
|
repairProgress: 0,
|
|
|
|
|
repairTotal: 0,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function wireUplinks(child: NetworkSwitch, parents: NetworkSwitch[], count: number): void {
|
|
|
|
|
if (parents.length === 0) return;
|
|
|
|
|
for (let i = 0; i < count; i++) {
|
|
|
|
|
const parent = parents[i % parents.length];
|
|
|
|
|
child.uplinkIds.push(parent.id);
|
|
|
|
|
if (!parent.downlinkIds.includes(child.id)) {
|
|
|
|
|
parent.downlinkIds.push(child.id);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
child.activeUplinks = count;
|
|
|
|
|
child.effectiveBandwidth = 1.0;
|
|
|
|
|
}
|
|
|
|
|
const DC_TIERS: SwitchTier[] = ['tor', 't1', 't2', 't3'];
|
|
|
|
|
|
|
|
|
|
export function emptyDCNetworkSummary(): DCNetworkSummary {
|
|
|
|
|
return {
|
|
|
|
|
switchIds: [], networkRackCount: 0,
|
|
|
|
|
totalByTier: {}, healthyByTier: {},
|
|
|
|
|
repairBatches: [], networkRackCount: 0,
|
|
|
|
|
racksDisconnected: 0, racksDegraded: 0,
|
|
|
|
|
averageBandwidth: 1, effectiveFlopsFraction: 1,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function emptyCampusNetworkSummary(): CampusNetworkSummary {
|
|
|
|
|
return { switchIds: [], totalT4: 0, healthyT4: 0, crossDCBandwidth: 1 };
|
|
|
|
|
return { totalT4: 0, healthyT4: 0, crossDCBandwidth: 1 };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function emptyClusterNetworkSummary(): ClusterNetworkSummary {
|
|
|
|
|
return { switchIds: [], totalT5: 0, healthyT5: 0, crossCampusBandwidth: 1 };
|
|
|
|
|
return { totalT5: 0, healthyT5: 0, crossCampusBandwidth: 1 };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function buildDCTopology(
|
|
|
|
|
function computeTopologyCounts(
|
|
|
|
|
computeRackCount: number,
|
|
|
|
|
dcTier: DCTier,
|
|
|
|
|
dcId: string,
|
|
|
|
|
registry: Record<string, NetworkSwitch>,
|
|
|
|
|
): DCNetworkSummary {
|
|
|
|
|
if (computeRackCount <= 0) return emptyDCNetworkSummary();
|
|
|
|
|
|
|
|
|
|
const switchIds: string[] = [];
|
|
|
|
|
|
|
|
|
|
const t3Count = T3_COUNT_PER_DC_TIER[dcTier];
|
|
|
|
|
const t3s: NetworkSwitch[] = [];
|
|
|
|
|
for (let i = 0; i < t3Count; i++) {
|
|
|
|
|
const sw = createSwitch('t3', dcId, null, null);
|
|
|
|
|
sw.totalUplinks = 0;
|
|
|
|
|
sw.activeUplinks = 0;
|
|
|
|
|
t3s.push(sw);
|
|
|
|
|
registry[sw.id] = sw;
|
|
|
|
|
switchIds.push(sw.id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
): Partial<Record<SwitchTier, number>> {
|
|
|
|
|
if (computeRackCount <= 0) return {};
|
|
|
|
|
const t1Count = Math.ceil(computeRackCount / SWITCH_TIER_CONFIGS.t1.fanOut);
|
|
|
|
|
const t2Count = Math.ceil(t1Count / SWITCH_TIER_CONFIGS.t2.fanOut);
|
|
|
|
|
|
|
|
|
|
const t2s: NetworkSwitch[] = [];
|
|
|
|
|
for (let i = 0; i < t2Count; i++) {
|
|
|
|
|
const sw = createSwitch('t2', dcId, null, null);
|
|
|
|
|
wireUplinks(sw, t3s, SWITCH_TIER_CONFIGS.t2.uplinkCount);
|
|
|
|
|
t2s.push(sw);
|
|
|
|
|
registry[sw.id] = sw;
|
|
|
|
|
switchIds.push(sw.id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const t1s: NetworkSwitch[] = [];
|
|
|
|
|
for (let i = 0; i < t1Count; i++) {
|
|
|
|
|
const sw = createSwitch('t1', dcId, null, null);
|
|
|
|
|
wireUplinks(sw, t2s, SWITCH_TIER_CONFIGS.t1.uplinkCount);
|
|
|
|
|
t1s.push(sw);
|
|
|
|
|
registry[sw.id] = sw;
|
|
|
|
|
switchIds.push(sw.id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (let i = 0; i < computeRackCount; i++) {
|
|
|
|
|
const sw = createSwitch('tor', dcId, null, null);
|
|
|
|
|
const primary = t1s[Math.floor(i / SWITCH_TIER_CONFIGS.t1.fanOut)];
|
|
|
|
|
const altIdx = (Math.floor(i / SWITCH_TIER_CONFIGS.t1.fanOut) + 1) % t1s.length;
|
|
|
|
|
const alt = t1s[altIdx];
|
|
|
|
|
if (t1s.length >= 2 && primary !== alt) {
|
|
|
|
|
wireUplinks(sw, [primary, alt], 2);
|
|
|
|
|
} else {
|
|
|
|
|
wireUplinks(sw, [primary], 2);
|
|
|
|
|
}
|
|
|
|
|
registry[sw.id] = sw;
|
|
|
|
|
switchIds.push(sw.id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const networkRackCount = estimateNetworkSlots(computeRackCount, dcTier);
|
|
|
|
|
return buildDCSummary(switchIds, networkRackCount, registry);
|
|
|
|
|
const t3Count = T3_COUNT_PER_DC_TIER[dcTier];
|
|
|
|
|
return { tor: computeRackCount, t1: t1Count, t2: t2Count, t3: t3Count };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function expandDCTopology(
|
|
|
|
|
existing: DCNetworkSummary,
|
|
|
|
|
newRackCount: number,
|
|
|
|
|
export function buildDCNetworkSummary(
|
|
|
|
|
computeRackCount: number,
|
|
|
|
|
dcTier: DCTier,
|
|
|
|
|
dcId: string,
|
|
|
|
|
registry: Record<string, NetworkSwitch>,
|
|
|
|
|
): DCNetworkSummary {
|
|
|
|
|
if (newRackCount <= 0) return existing;
|
|
|
|
|
|
|
|
|
|
const currentTorCount = existing.totalByTier?.tor ?? 0;
|
|
|
|
|
const targetTorCount = currentTorCount + newRackCount;
|
|
|
|
|
|
|
|
|
|
const t1s = existing.switchIds.map(id => registry[id]).filter((s): s is NetworkSwitch => !!s && s.tier === 't1');
|
|
|
|
|
const t2s = existing.switchIds.map(id => registry[id]).filter((s): s is NetworkSwitch => !!s && s.tier === 't2');
|
|
|
|
|
const t3s = existing.switchIds.map(id => registry[id]).filter((s): s is NetworkSwitch => !!s && s.tier === 't3');
|
|
|
|
|
|
|
|
|
|
const newIds = [...existing.switchIds];
|
|
|
|
|
|
|
|
|
|
const neededT1 = Math.ceil(targetTorCount / SWITCH_TIER_CONFIGS.t1.fanOut);
|
|
|
|
|
const neededT2 = Math.ceil(neededT1 / SWITCH_TIER_CONFIGS.t2.fanOut);
|
|
|
|
|
|
|
|
|
|
while (t2s.length < neededT2) {
|
|
|
|
|
const sw = createSwitch('t2', dcId, null, null);
|
|
|
|
|
wireUplinks(sw, t3s, SWITCH_TIER_CONFIGS.t2.uplinkCount);
|
|
|
|
|
t2s.push(sw);
|
|
|
|
|
registry[sw.id] = sw;
|
|
|
|
|
newIds.push(sw.id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
while (t1s.length < neededT1) {
|
|
|
|
|
const sw = createSwitch('t1', dcId, null, null);
|
|
|
|
|
wireUplinks(sw, t2s, SWITCH_TIER_CONFIGS.t1.uplinkCount);
|
|
|
|
|
t1s.push(sw);
|
|
|
|
|
registry[sw.id] = sw;
|
|
|
|
|
newIds.push(sw.id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (let i = 0; i < newRackCount; i++) {
|
|
|
|
|
const torIdx = currentTorCount + i;
|
|
|
|
|
const sw = createSwitch('tor', dcId, null, null);
|
|
|
|
|
const primary = t1s[Math.floor(torIdx / SWITCH_TIER_CONFIGS.t1.fanOut)];
|
|
|
|
|
const altIdx = (Math.floor(torIdx / SWITCH_TIER_CONFIGS.t1.fanOut) + 1) % t1s.length;
|
|
|
|
|
const alt = t1s[altIdx];
|
|
|
|
|
if (t1s.length >= 2 && primary !== alt) {
|
|
|
|
|
wireUplinks(sw, [primary, alt], 2);
|
|
|
|
|
} else {
|
|
|
|
|
wireUplinks(sw, [primary], 2);
|
|
|
|
|
}
|
|
|
|
|
registry[sw.id] = sw;
|
|
|
|
|
newIds.push(sw.id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const networkRackCount = estimateNetworkSlots(targetTorCount, dcTier);
|
|
|
|
|
return buildDCSummary(newIds, networkRackCount, registry);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function shrinkDCTopology(
|
|
|
|
|
existing: DCNetworkSummary,
|
|
|
|
|
removeCount: number,
|
|
|
|
|
dcTier: DCTier,
|
|
|
|
|
registry: Record<string, NetworkSwitch>,
|
|
|
|
|
): DCNetworkSummary {
|
|
|
|
|
if (removeCount <= 0) return existing;
|
|
|
|
|
|
|
|
|
|
const torIds = existing.switchIds.filter(id => registry[id]?.tier === 'tor');
|
|
|
|
|
const toRemove = new Set(torIds.slice(-removeCount));
|
|
|
|
|
|
|
|
|
|
for (const torId of toRemove) {
|
|
|
|
|
const tor = registry[torId];
|
|
|
|
|
if (!tor) continue;
|
|
|
|
|
for (const upId of tor.uplinkIds) {
|
|
|
|
|
const parent = registry[upId];
|
|
|
|
|
if (parent) parent.downlinkIds = parent.downlinkIds.filter(id => id !== torId);
|
|
|
|
|
}
|
|
|
|
|
delete registry[torId];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const remainingIds = existing.switchIds.filter(id => !toRemove.has(id));
|
|
|
|
|
const remainingTors = remainingIds.filter(id => registry[id]?.tier === 'tor').length;
|
|
|
|
|
return buildDCSummary(remainingIds, estimateNetworkSlots(remainingTors, dcTier), registry);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function computeRackBandwidth(tor: NetworkSwitch, registry: Record<string, NetworkSwitch>): number {
|
|
|
|
|
if (tor.status !== 'healthy') return 0;
|
|
|
|
|
|
|
|
|
|
let minBW = tor.totalUplinks > 0 ? tor.activeUplinks / tor.totalUplinks : 1;
|
|
|
|
|
if (minBW === 0) return 0;
|
|
|
|
|
|
|
|
|
|
const visited = new Set<string>();
|
|
|
|
|
let current = tor.uplinkIds.filter(id => {
|
|
|
|
|
const sw = registry[id];
|
|
|
|
|
return sw && sw.status === 'healthy';
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
while (current.length > 0) {
|
|
|
|
|
let tierBW = 1;
|
|
|
|
|
const next: string[] = [];
|
|
|
|
|
for (const sid of current) {
|
|
|
|
|
if (visited.has(sid)) continue;
|
|
|
|
|
visited.add(sid);
|
|
|
|
|
const sw = registry[sid];
|
|
|
|
|
if (!sw || sw.status !== 'healthy') continue;
|
|
|
|
|
const bw = sw.totalUplinks > 0 ? sw.activeUplinks / sw.totalUplinks : 1;
|
|
|
|
|
tierBW = Math.min(tierBW, bw);
|
|
|
|
|
for (const upId of sw.uplinkIds) {
|
|
|
|
|
if (registry[upId]?.status === 'healthy') next.push(upId);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
minBW = Math.min(minBW, tierBW);
|
|
|
|
|
if (minBW === 0) return 0;
|
|
|
|
|
current = next;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return minBW;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function buildDCSummary(
|
|
|
|
|
switchIds: string[],
|
|
|
|
|
networkRackCount: number,
|
|
|
|
|
registry: Record<string, NetworkSwitch>,
|
|
|
|
|
): DCNetworkSummary {
|
|
|
|
|
const totalByTier: Partial<Record<SwitchTier, number>> = {};
|
|
|
|
|
const healthyByTier: Partial<Record<SwitchTier, number>> = {};
|
|
|
|
|
let disconnected = 0;
|
|
|
|
|
let degraded = 0;
|
|
|
|
|
let bwSum = 0;
|
|
|
|
|
let torCount = 0;
|
|
|
|
|
|
|
|
|
|
for (const sid of switchIds) {
|
|
|
|
|
const sw = registry[sid];
|
|
|
|
|
if (!sw) continue;
|
|
|
|
|
totalByTier[sw.tier] = (totalByTier[sw.tier] ?? 0) + 1;
|
|
|
|
|
if (sw.status === 'healthy') healthyByTier[sw.tier] = (healthyByTier[sw.tier] ?? 0) + 1;
|
|
|
|
|
if (sw.tier === 'tor') {
|
|
|
|
|
torCount++;
|
|
|
|
|
const bw = computeRackBandwidth(sw, registry);
|
|
|
|
|
bwSum += bw;
|
|
|
|
|
if (bw === 0) disconnected++;
|
|
|
|
|
else if (bw < 1) degraded++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const avgBW = torCount > 0 ? bwSum / torCount : 1;
|
|
|
|
|
if (computeRackCount <= 0) return emptyDCNetworkSummary();
|
|
|
|
|
const totalByTier = computeTopologyCounts(computeRackCount, dcTier);
|
|
|
|
|
const healthyByTier = { ...totalByTier };
|
|
|
|
|
return {
|
|
|
|
|
switchIds, networkRackCount, totalByTier, healthyByTier,
|
|
|
|
|
racksDisconnected: disconnected, racksDegraded: degraded,
|
|
|
|
|
averageBandwidth: avgBW, effectiveFlopsFraction: avgBW,
|
|
|
|
|
totalByTier, healthyByTier,
|
|
|
|
|
repairBatches: [],
|
|
|
|
|
networkRackCount: estimateNetworkSlots(computeRackCount, dcTier),
|
|
|
|
|
racksDisconnected: 0, racksDegraded: 0,
|
|
|
|
|
averageBandwidth: 1, effectiveFlopsFraction: 1,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// --- Network Tick (failure rolls + repair) ---
|
|
|
|
|
export function expandDCNetwork(
|
|
|
|
|
existing: DCNetworkSummary,
|
|
|
|
|
addedRacks: number,
|
|
|
|
|
dcTier: DCTier,
|
|
|
|
|
): DCNetworkSummary {
|
|
|
|
|
if (addedRacks <= 0) return existing;
|
|
|
|
|
const oldTor = existing.totalByTier.tor ?? 0;
|
|
|
|
|
const newTor = oldTor + addedRacks;
|
|
|
|
|
const newTotal = computeTopologyCounts(newTor, dcTier);
|
|
|
|
|
const healthyByTier: Partial<Record<SwitchTier, number>> = {};
|
|
|
|
|
for (const tier of DC_TIERS) {
|
|
|
|
|
const oldTotal = existing.totalByTier[tier] ?? 0;
|
|
|
|
|
const oldHealthy = existing.healthyByTier[tier] ?? 0;
|
|
|
|
|
const added = (newTotal[tier] ?? 0) - oldTotal;
|
|
|
|
|
healthyByTier[tier] = oldHealthy + Math.max(0, added);
|
|
|
|
|
}
|
|
|
|
|
const summary: DCNetworkSummary = {
|
|
|
|
|
...existing,
|
|
|
|
|
totalByTier: newTotal,
|
|
|
|
|
healthyByTier,
|
|
|
|
|
networkRackCount: estimateNetworkSlots(newTor, dcTier),
|
|
|
|
|
};
|
|
|
|
|
return recomputeBandwidth(summary);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function processNetworkTick(
|
|
|
|
|
registry: Record<string, NetworkSwitch>,
|
|
|
|
|
export function shrinkDCNetwork(
|
|
|
|
|
existing: DCNetworkSummary,
|
|
|
|
|
removedRacks: number,
|
|
|
|
|
dcTier: DCTier,
|
|
|
|
|
): DCNetworkSummary {
|
|
|
|
|
if (removedRacks <= 0) return existing;
|
|
|
|
|
const oldTor = existing.totalByTier.tor ?? 0;
|
|
|
|
|
const newTor = Math.max(0, oldTor - removedRacks);
|
|
|
|
|
if (newTor === 0) return emptyDCNetworkSummary();
|
|
|
|
|
const newTotal = computeTopologyCounts(newTor, dcTier);
|
|
|
|
|
const healthyByTier: Partial<Record<SwitchTier, number>> = {};
|
|
|
|
|
for (const tier of DC_TIERS) {
|
|
|
|
|
const nt = newTotal[tier] ?? 0;
|
|
|
|
|
const oh = existing.healthyByTier[tier] ?? 0;
|
|
|
|
|
healthyByTier[tier] = Math.min(oh, nt);
|
|
|
|
|
}
|
|
|
|
|
const repairBatches = existing.repairBatches.filter(b => {
|
|
|
|
|
const nt = newTotal[b.tier] ?? 0;
|
|
|
|
|
const nh = healthyByTier[b.tier] ?? 0;
|
|
|
|
|
return nh < nt;
|
|
|
|
|
});
|
|
|
|
|
const summary: DCNetworkSummary = {
|
|
|
|
|
...existing,
|
|
|
|
|
totalByTier: newTotal,
|
|
|
|
|
healthyByTier,
|
|
|
|
|
repairBatches,
|
|
|
|
|
networkRackCount: estimateNetworkSlots(newTor, dcTier),
|
|
|
|
|
};
|
|
|
|
|
return recomputeBandwidth(summary);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function computeAggregateBandwidth(
|
|
|
|
|
summary: DCNetworkSummary,
|
|
|
|
|
redundancyBonus: number,
|
|
|
|
|
): number {
|
|
|
|
|
let minBW = 1;
|
|
|
|
|
for (const tier of DC_TIERS) {
|
|
|
|
|
const total = summary.totalByTier[tier] ?? 0;
|
|
|
|
|
if (total === 0) continue;
|
|
|
|
|
const healthy = summary.healthyByTier[tier] ?? 0;
|
|
|
|
|
const tierBW = Math.min(1, (healthy + redundancyBonus) / total);
|
|
|
|
|
if (tierBW < minBW) minBW = tierBW;
|
|
|
|
|
}
|
|
|
|
|
return minBW;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function recomputeBandwidth(summary: DCNetworkSummary, redundancyBonus = 0): DCNetworkSummary {
|
|
|
|
|
const avgBW = computeAggregateBandwidth(summary, redundancyBonus);
|
|
|
|
|
const torTotal = summary.totalByTier.tor ?? 0;
|
|
|
|
|
const torHealthy = summary.healthyByTier.tor ?? 0;
|
|
|
|
|
const torFailed = torTotal - torHealthy;
|
|
|
|
|
const disconnected = avgBW === 0 ? torTotal : torFailed;
|
|
|
|
|
const degraded = avgBW > 0 && avgBW < 1 ? Math.ceil(torTotal * (1 - avgBW)) - disconnected : 0;
|
|
|
|
|
return {
|
|
|
|
|
...summary,
|
|
|
|
|
averageBandwidth: avgBW,
|
|
|
|
|
effectiveFlopsFraction: avgBW,
|
|
|
|
|
racksDisconnected: Math.max(0, disconnected),
|
|
|
|
|
racksDegraded: Math.max(0, degraded),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function processNetworkForDC(
|
|
|
|
|
summary: DCNetworkSummary,
|
|
|
|
|
networkResearchBonus: number,
|
|
|
|
|
opsEff: number,
|
|
|
|
|
repairSpeedBonus: number,
|
|
|
|
|
hotStandbyTicks: number,
|
|
|
|
|
redundancyBonus: number,
|
|
|
|
|
): { switchRepairCosts: number; notifications: TickNotification[]; dirtyDCs: Set<string> } {
|
|
|
|
|
): { summary: DCNetworkSummary; costs: number; notifications: TickNotification[] } {
|
|
|
|
|
const torTotal = summary.totalByTier.tor ?? 0;
|
|
|
|
|
if (torTotal === 0) return { summary, costs: 0, notifications: [] };
|
|
|
|
|
|
|
|
|
|
let costs = 0;
|
|
|
|
|
const notifications: TickNotification[] = [];
|
|
|
|
|
let switchRepairCosts = 0;
|
|
|
|
|
const dirtyDCs = new Set<string>();
|
|
|
|
|
const healthyByTier = { ...summary.healthyByTier };
|
|
|
|
|
let dirty = false;
|
|
|
|
|
|
|
|
|
|
const healthyByTier: Partial<Record<SwitchTier, NetworkSwitch[]>> = {};
|
|
|
|
|
const repairing: NetworkSwitch[] = [];
|
|
|
|
|
|
|
|
|
|
for (const sw of Object.values(registry)) {
|
|
|
|
|
if (sw.status === 'healthy') {
|
|
|
|
|
(healthyByTier[sw.tier] ??= []).push(sw);
|
|
|
|
|
} else if (sw.status === 'repairing') {
|
|
|
|
|
repairing.push(sw);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const tiers: SwitchTier[] = ['tor', 't1', 't2', 't3', 't4', 't5'];
|
|
|
|
|
const newlyFailed: NetworkSwitch[] = [];
|
|
|
|
|
|
|
|
|
|
for (const tier of tiers) {
|
|
|
|
|
const healthy = healthyByTier[tier];
|
|
|
|
|
if (!healthy || healthy.length === 0) continue;
|
|
|
|
|
for (const tier of DC_TIERS) {
|
|
|
|
|
const healthy = healthyByTier[tier] ?? 0;
|
|
|
|
|
if (healthy <= 0) continue;
|
|
|
|
|
const rate = SWITCH_TIER_CONFIGS[tier].failureRatePerTick * (1 - networkResearchBonus);
|
|
|
|
|
const count = binomialSample(healthy.length, rate);
|
|
|
|
|
if (count > 0) {
|
|
|
|
|
const shuffled = [...healthy].sort(() => Math.random() - 0.5);
|
|
|
|
|
for (let i = 0; i < count; i++) {
|
|
|
|
|
const sw = shuffled[i];
|
|
|
|
|
const baseRepair = SWITCH_TIER_CONFIGS[tier].repairBaseTicks;
|
|
|
|
|
const repairTime = hotStandbyTicks > 0
|
|
|
|
|
? hotStandbyTicks
|
|
|
|
|
: baseRepair * (1 - repairSpeedBonus);
|
|
|
|
|
sw.status = 'repairing';
|
|
|
|
|
sw.repairProgress = 0;
|
|
|
|
|
sw.repairTotal = repairTime;
|
|
|
|
|
newlyFailed.push(sw);
|
|
|
|
|
if (sw.dcId) dirtyDCs.add(sw.dcId);
|
|
|
|
|
switchRepairCosts += SWITCH_TIER_CONFIGS[tier].baseCost * SWITCH_REPAIR_COST_FRACTION;
|
|
|
|
|
const failed = binomialSample(healthy, rate);
|
|
|
|
|
if (failed > 0) {
|
|
|
|
|
healthyByTier[tier] = healthy - failed;
|
|
|
|
|
const baseRepair = SWITCH_TIER_CONFIGS[tier].repairBaseTicks;
|
|
|
|
|
const repairTime = hotStandbyTicks > 0
|
|
|
|
|
? hotStandbyTicks
|
|
|
|
|
: baseRepair * (1 - repairSpeedBonus);
|
|
|
|
|
summary.repairBatches.push({ tier, count: failed, ticksRemaining: repairTime });
|
|
|
|
|
costs += SWITCH_TIER_CONFIGS[tier].baseCost * SWITCH_REPAIR_COST_FRACTION * failed;
|
|
|
|
|
dirty = true;
|
|
|
|
|
|
|
|
|
|
if (tier === 't3') {
|
|
|
|
|
notifications.push({ title: 'Core Network Failure', message: `Tier-3 core switch failed — potential DC disconnect!`, type: 'danger' });
|
|
|
|
|
} else if (tier === 't2') {
|
|
|
|
|
notifications.push({ title: 'Network Switch Failure', message: `Tier-2 spine switch failed — racks may be degraded.`, type: 'warning' });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const sw of repairing) {
|
|
|
|
|
sw.repairProgress += 1 + opsEff * 0.05;
|
|
|
|
|
if (sw.repairProgress >= sw.repairTotal) {
|
|
|
|
|
sw.status = 'healthy';
|
|
|
|
|
sw.repairProgress = 0;
|
|
|
|
|
sw.repairTotal = 0;
|
|
|
|
|
if (sw.dcId) dirtyDCs.add(sw.dcId);
|
|
|
|
|
const remainingBatches: RepairBatch[] = [];
|
|
|
|
|
for (const batch of summary.repairBatches) {
|
|
|
|
|
const newTicks = batch.ticksRemaining - (1 + opsEff * 0.05);
|
|
|
|
|
if (newTicks <= 0) {
|
|
|
|
|
healthyByTier[batch.tier] = Math.min(
|
|
|
|
|
summary.totalByTier[batch.tier] ?? 0,
|
|
|
|
|
(healthyByTier[batch.tier] ?? 0) + batch.count,
|
|
|
|
|
);
|
|
|
|
|
dirty = true;
|
|
|
|
|
} else {
|
|
|
|
|
remainingBatches.push({ ...batch, ticksRemaining: newTicks });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (dirtyDCs.size > 0) {
|
|
|
|
|
for (const sw of Object.values(registry)) {
|
|
|
|
|
if (sw.uplinkIds.length === 0) continue;
|
|
|
|
|
if (sw.dcId && !dirtyDCs.has(sw.dcId)) continue;
|
|
|
|
|
let active = 0;
|
|
|
|
|
for (const upId of sw.uplinkIds) {
|
|
|
|
|
if (registry[upId]?.status === 'healthy') active++;
|
|
|
|
|
}
|
|
|
|
|
sw.activeUplinks = active;
|
|
|
|
|
sw.effectiveBandwidth = sw.totalUplinks > 0 ? Math.min(1, (active + redundancyBonus) / sw.totalUplinks) : 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (!dirty) return { summary: { ...summary, repairBatches: remainingBatches }, costs, notifications };
|
|
|
|
|
|
|
|
|
|
for (const sw of newlyFailed) {
|
|
|
|
|
if (sw.tier === 't3') {
|
|
|
|
|
notifications.push({ title: 'Core Network Failure', message: `Tier-3 core switch failed — potential DC disconnect!`, type: 'danger' });
|
|
|
|
|
} else if (sw.tier === 't4') {
|
|
|
|
|
notifications.push({ title: 'Campus Network Failure', message: `Tier-4 campus switch failed — cross-DC degradation!`, type: 'danger' });
|
|
|
|
|
} else if (sw.tier === 't2') {
|
|
|
|
|
notifications.push({ title: 'Network Switch Failure', message: `Tier-2 spine switch failed — racks may be degraded.`, type: 'warning' });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return { switchRepairCosts, notifications, dirtyDCs };
|
|
|
|
|
const updated: DCNetworkSummary = {
|
|
|
|
|
...summary,
|
|
|
|
|
healthyByTier,
|
|
|
|
|
repairBatches: remainingBatches,
|
|
|
|
|
};
|
|
|
|
|
return { summary: recomputeBandwidth(updated, redundancyBonus), costs, notifications };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// --- Interconnect Training Multiplier ---
|
|
|
|
@@ -476,14 +321,6 @@ export function processInfrastructure(state: GameState, researchBonuses?: Resear
|
|
|
|
|
const hotStandbyTicks = state.research.completedResearch.includes('network-hot-standby') ? 5 : 0;
|
|
|
|
|
const redundancyBonus = state.research.completedResearch.includes('network-redundancy') ? 1 : 0;
|
|
|
|
|
|
|
|
|
|
// Mutate registry in-place — infrastructure returns a new state anyway
|
|
|
|
|
const registry = state.infrastructure.switchRegistry;
|
|
|
|
|
|
|
|
|
|
// Process network failures/repairs globally
|
|
|
|
|
const netResult = processNetworkTick(registry, networkResearchBonus, opsEff, repairSpeedBonus, hotStandbyTicks, redundancyBonus);
|
|
|
|
|
repairCosts += netResult.switchRepairCosts;
|
|
|
|
|
if (netResult.notifications.length > 0) notifications.push(...netResult.notifications);
|
|
|
|
|
|
|
|
|
|
let totalFlops = 0;
|
|
|
|
|
let totalTrainingFlops = 0;
|
|
|
|
|
let totalInferenceFlops = 0;
|
|
|
|
@@ -541,8 +378,6 @@ export function processInfrastructure(state: GameState, researchBonuses?: Resear
|
|
|
|
|
if (rs.progress >= rs.total) {
|
|
|
|
|
if (rs.phase === 'decommissioning') {
|
|
|
|
|
const installTotal = cohortStageTotal('installation', rs.toSkuId, rs.racksRemaining);
|
|
|
|
|
// Clear DC topology on retrofit
|
|
|
|
|
for (const sid of dc.networkSummary.switchIds) delete registry[sid];
|
|
|
|
|
return {
|
|
|
|
|
...dc,
|
|
|
|
|
computeRacksOnline: 0,
|
|
|
|
@@ -636,10 +471,11 @@ export function processInfrastructure(state: GameState, researchBonuses?: Resear
|
|
|
|
|
// Expand topology for newly onlined racks
|
|
|
|
|
let networkSummary = dc.networkSummary;
|
|
|
|
|
if (racksJustOnlined > 0) {
|
|
|
|
|
if (networkSummary.switchIds.length === 0) {
|
|
|
|
|
networkSummary = buildDCTopology(computeRacksOnline, dc.tier, dc.id, registry);
|
|
|
|
|
const torTotal = networkSummary.totalByTier.tor ?? 0;
|
|
|
|
|
if (torTotal === 0) {
|
|
|
|
|
networkSummary = buildDCNetworkSummary(computeRacksOnline, dc.tier);
|
|
|
|
|
} else {
|
|
|
|
|
networkSummary = expandDCTopology(networkSummary, racksJustOnlined, dc.tier, dc.id, registry);
|
|
|
|
|
networkSummary = expandDCNetwork(networkSummary, racksJustOnlined, dc.tier);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -660,18 +496,20 @@ export function processInfrastructure(state: GameState, researchBonuses?: Resear
|
|
|
|
|
stageTotal: cohortStageTotal('repair', dc.rackSkuId, prodFailures),
|
|
|
|
|
repairCount: 0,
|
|
|
|
|
});
|
|
|
|
|
networkSummary = shrinkDCTopology(networkSummary, prodFailures, dc.tier, registry);
|
|
|
|
|
networkSummary = shrinkDCNetwork(networkSummary, prodFailures, dc.tier);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
repairCosts += dcRepairCosts;
|
|
|
|
|
|
|
|
|
|
// Recompute DC network summary after failures/repairs (only if this DC's switches changed)
|
|
|
|
|
if (netResult.dirtyDCs.has(dc.id) && networkSummary.switchIds.length > 0) {
|
|
|
|
|
networkSummary = buildDCSummary(
|
|
|
|
|
networkSummary.switchIds, networkSummary.networkRackCount, registry,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
// Process per-DC network failures and repairs (aggregate model)
|
|
|
|
|
const netResult = processNetworkForDC(
|
|
|
|
|
networkSummary, networkResearchBonus, opsEff,
|
|
|
|
|
repairSpeedBonus, hotStandbyTicks, redundancyBonus,
|
|
|
|
|
);
|
|
|
|
|
networkSummary = netResult.summary;
|
|
|
|
|
repairCosts += netResult.costs;
|
|
|
|
|
if (netResult.notifications.length > 0) notifications.push(...netResult.notifications);
|
|
|
|
|
|
|
|
|
|
// Rackdown: detect recovery (previously disconnected racks now have connectivity)
|
|
|
|
|
const prevDisconnected = dc.networkSummary.racksDisconnected;
|
|
|
|
@@ -680,7 +518,7 @@ export function processInfrastructure(state: GameState, researchBonuses?: Resear
|
|
|
|
|
if (currDisconnected < prevDisconnected && dc.rackSkuId) {
|
|
|
|
|
const recovered = prevDisconnected - currDisconnected;
|
|
|
|
|
computeRacksOnline -= recovered;
|
|
|
|
|
networkSummary = shrinkDCTopology(networkSummary, recovered, dc.tier, registry);
|
|
|
|
|
networkSummary = shrinkDCNetwork(networkSummary, recovered, dc.tier);
|
|
|
|
|
updatedCohorts.push({
|
|
|
|
|
id: `netrecovery-${dc.id}-${Date.now()}`,
|
|
|
|
|
count: recovered, skuId: dc.rackSkuId,
|
|
|
|
@@ -688,10 +526,6 @@ export function processInfrastructure(state: GameState, researchBonuses?: Resear
|
|
|
|
|
stageTotal: cohortStageTotal('testing', dc.rackSkuId, recovered),
|
|
|
|
|
repairCount: 0,
|
|
|
|
|
});
|
|
|
|
|
// Recompute summary after shrink
|
|
|
|
|
networkSummary = buildDCSummary(
|
|
|
|
|
networkSummary.switchIds, networkSummary.networkRackCount, registry,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Compute DC aggregates
|
|
|
|
@@ -789,8 +623,6 @@ export function processInfrastructure(state: GameState, researchBonuses?: Resear
|
|
|
|
|
if (totalRacks <= 0) return dc;
|
|
|
|
|
const oldSku = RACK_SKU_CONFIGS[dc.rackSkuId as RackSkuId];
|
|
|
|
|
const decommTicks = Math.ceil(oldSku.pipelineTimeTicks.installation * (1 + COHORT_SCALE_FACTOR * totalRacks));
|
|
|
|
|
// Clear topology on retrofit start
|
|
|
|
|
for (const sid of dc.networkSummary.switchIds) delete registry[sid];
|
|
|
|
|
return {
|
|
|
|
|
...dc,
|
|
|
|
|
status: 'retrofitting' as const,
|
|
|
|
@@ -827,7 +659,6 @@ export function processInfrastructure(state: GameState, researchBonuses?: Resear
|
|
|
|
|
return {
|
|
|
|
|
infrastructure: {
|
|
|
|
|
clusters,
|
|
|
|
|
switchRegistry: registry,
|
|
|
|
|
totalFlops,
|
|
|
|
|
totalTrainingFlops,
|
|
|
|
|
totalInferenceFlops,
|
|
|
|
|