Files
josh 23c689aa5b
CI / Lint + build + test (push) Failing after 1m57s
Release / release (push) Has been cancelled
deep profile + threshold gating + firmware stage + Burn super-stage
Ships all five phases of the deep-profile overhaul together. Runs now
carry a profile (quick/deep/soak); every profile walks the same
11-stage order — Inventory → Firmware → SpecValidate → SMART →
CPUStress → Storage → Network → Burn → GPU → PSU → Reporting —
with only per-stage durations and concurrency scaled.

Phase 1: profiles.ProfileRegistry loaded from vetting.yaml; runs.profile
column + CreateWithProfile; threshold table + evaluator seeded per-run
from the shared vetting.thresholds block; breach flips result at
/sensor + /result.

Phase 2: upgraded CPUStress (stress-ng --cpu-method=all --verify +
EDAC/MCE poll), Storage (fio --verify=md5 + SMART start/end delta),
Network (sustained iperf + /proc/net/dev deltas) with per-profile
knobs from Deps.

Phase 3: Burn super-stage with goroutine fan-out for CPU + memory +
fio + iperf, PSU rails sampled across the Burn window, SensorMux
(2 s flush, 500-sample cap) to absorb backpressure.

Phase 4: Firmware stage + firmware_snapshots table; probes dmidecode
(BIOS), ipmitool (BMC), ethtool -i (NIC), nvme (sysfs + id-ctrl),
lspci (HBA), /proc/cpuinfo (microcode). spec.DiffFirmware folds into
SpecValidate with pin-by-identifier and fan-out-across-component
matching; mismatches park the run in FailedHolding.

Phase 5: profile radio on the host start form, profile chip on the
run header, Firmware section in the HTML report, coverage artifact
uploaded from CI, agent/tests/fakes/ scaffold with Deps.LookPath
seam + stress_ng and dmidecode example fakes.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-04-18 22:50:57 -04:00

229 lines
7.7 KiB
Go

package templates
import (
"strings"
"testing"
"vetting/internal/model"
)
// node indexes for the default pipeline layout: pre-stages (3) + stage
// rows (11) + terminal Completed (1) = 15 nodes.
const (
idxQueued = 0
idxWaitingReboot = 1
idxBooting = 2
idxInventory = 3
idxFirmware = 4
idxSpecValidate = 5
idxSMART = 6
idxCPUStress = 7
idxStorage = 8
idxNetwork = 9
idxBurn = 10
idxGPU = 11
idxPSU = 12
idxReporting = 13
idxCompleted = 14
)
// seedStages returns a fresh all-pending stage slice in the canonical order.
func seedStages() []model.Stage {
names := []string{"Inventory", "Firmware", "SpecValidate", "SMART", "CPUStress", "Storage", "Network", "Burn", "GPU", "PSU", "Reporting"}
out := make([]model.Stage, len(names))
for i, n := range names {
out[i] = model.Stage{Name: n, Ordinal: i, State: model.StagePending}
}
return out
}
func TestBuildPipeline_NoRun(t *testing.T) {
nodes := BuildPipeline(nil, nil)
// Ghost pipeline: 3 pre-stages + 10 stage ghosts + 1 terminal = 14
// nodes, all pending.
if len(nodes) != 15 {
t.Fatalf("len = %d, want 15", len(nodes))
}
for i, n := range nodes {
if n.State != "pending" {
t.Errorf("node %d (%s) state = %q, want pending", i, n.Name, n.State)
}
}
}
// TestBuildPipeline_GhostStagesBeforeClaim models the real WaitingReboot
// case: the run exists but agent hasn't called /claim yet, so there are
// no stage rows. Pipeline must still render all 9 stage nodes as ghosts
// so the operator sees the full timeline ahead of them.
func TestBuildPipeline_GhostStagesBeforeClaim(t *testing.T) {
run := &model.Run{State: model.StateWaitingReboot}
nodes := BuildPipeline(run, nil)
if len(nodes) != 15 {
t.Fatalf("len = %d, want 15", len(nodes))
}
if nodes[idxQueued].State != "passed" {
t.Errorf("Queued = %q, want passed", nodes[idxQueued].State)
}
if nodes[idxWaitingReboot].State != "running" {
t.Errorf("WaitingReboot = %q, want running", nodes[idxWaitingReboot].State)
}
// All 11 stage ghosts must be pending — nothing has started yet.
for i := idxInventory; i <= idxReporting; i++ {
if nodes[i].State != "pending" {
t.Errorf("%s (ghost) = %q, want pending", nodes[i].Name, nodes[i].State)
}
}
if nodes[idxCompleted].State != "pending" {
t.Errorf("Completed = %q, want pending", nodes[idxCompleted].State)
}
}
// TestBuildPipeline_GhostStagesDuringStage models the in-flight case
// with only some stage rows seeded: later stages must still appear as
// pending ghosts rather than silently disappearing.
func TestBuildPipeline_GhostStagesDuringStage(t *testing.T) {
run := &model.Run{State: model.StateSMART}
// Only Inventory + Firmware + SpecValidate seeded; SMART onwards are ghosts.
stages := []model.Stage{
{Name: "Inventory", Ordinal: 0, State: model.StagePassed},
{Name: "Firmware", Ordinal: 1, State: model.StagePassed},
{Name: "SpecValidate", Ordinal: 2, State: model.StagePassed},
}
nodes := BuildPipeline(run, stages)
if len(nodes) != 15 {
t.Fatalf("len = %d, want 15", len(nodes))
}
if nodes[idxSMART].State != "running" {
t.Errorf("SMART (ghost) = %q, want running", nodes[idxSMART].State)
}
for _, i := range []int{idxCPUStress, idxStorage, idxNetwork, idxBurn, idxGPU, idxPSU, idxReporting} {
if nodes[i].State != "pending" {
t.Errorf("%s (ghost) = %q, want pending", nodes[i].Name, nodes[i].State)
}
}
}
func TestBuildPipeline_Running(t *testing.T) {
run := &model.Run{State: model.StateSMART}
stages := seedStages()
stages[0].State = model.StagePassed // Inventory
stages[1].State = model.StagePassed // Firmware
stages[2].State = model.StagePassed // SpecValidate
stages[3].State = model.StageRunning // SMART
nodes := BuildPipeline(run, stages)
if len(nodes) != 15 {
t.Fatalf("len = %d, want 15", len(nodes))
}
// Pre-stages are all past for a run that has reached SMART.
for i := idxQueued; i <= idxBooting; i++ {
if nodes[i].State != "passed" {
t.Errorf("prestage %s = %q, want passed", nodes[i].Name, nodes[i].State)
}
}
if nodes[idxInventory].State != "passed" {
t.Errorf("Inventory = %q, want passed", nodes[idxInventory].State)
}
if nodes[idxSpecValidate].State != "passed" {
t.Errorf("SpecValidate = %q, want passed", nodes[idxSpecValidate].State)
}
if nodes[idxSMART].State != "running" {
t.Errorf("SMART = %q, want running", nodes[idxSMART].State)
}
if nodes[idxCPUStress].State != "pending" {
t.Errorf("CPUStress = %q, want pending", nodes[idxCPUStress].State)
}
if nodes[idxCompleted].State != "pending" {
t.Errorf("Completed = %q, want pending", nodes[idxCompleted].State)
}
}
func TestBuildPipeline_Failed(t *testing.T) {
run := &model.Run{State: model.StateFailedHolding, FailedStage: "Storage"}
stages := seedStages()
for i := 0; i <= 4; i++ {
stages[i].State = model.StagePassed
}
stages[5].State = model.StageFailed // Storage
nodes := BuildPipeline(run, stages)
// Pre-stages are past a run that reached Storage.
for i := idxQueued; i <= idxBooting; i++ {
if nodes[i].State != "passed" {
t.Errorf("prestage %s = %q, want passed", nodes[i].Name, nodes[i].State)
}
}
if nodes[idxStorage].State != "failed" {
t.Errorf("Storage = %q, want failed", nodes[idxStorage].State)
}
for _, i := range []int{idxNetwork, idxBurn, idxGPU, idxPSU, idxReporting} {
if nodes[i].State != "skipped" {
t.Errorf("%s = %q, want skipped", nodes[i].Name, nodes[i].State)
}
}
if nodes[idxCompleted].State != "pending" {
t.Errorf("Completed = %q, want pending on failure", nodes[idxCompleted].State)
}
}
func TestBuildPipeline_Completed(t *testing.T) {
run := &model.Run{State: model.StateCompleted}
stages := seedStages()
for i := range stages {
stages[i].State = model.StagePassed
}
nodes := BuildPipeline(run, stages)
for i, n := range nodes {
if n.State != "passed" {
t.Errorf("node %d (%s) state = %q, want passed", i, n.Name, n.State)
}
}
}
func TestBuildPipeline_QueuedNow(t *testing.T) {
run := &model.Run{State: model.StateQueued}
nodes := BuildPipeline(run, seedStages())
if nodes[idxQueued].State != "running" {
t.Errorf("Queued = %q, want running", nodes[idxQueued].State)
}
if nodes[idxWaitingReboot].State != "pending" {
t.Errorf("WaitingReboot = %q, want pending", nodes[idxWaitingReboot].State)
}
}
// TestBuildPipeline_PreStageRunning_WaitingReboot confirms the pre-stage
// node for WaitingReboot lights up while the run sits there — the new
// happy-path state must map onto its pipeline slot.
func TestBuildPipeline_PreStageRunning_WaitingReboot(t *testing.T) {
run := &model.Run{State: model.StateWaitingReboot}
nodes := BuildPipeline(run, seedStages())
if nodes[idxQueued].State != "passed" {
t.Errorf("Queued = %q, want passed", nodes[idxQueued].State)
}
if nodes[idxWaitingReboot].State != "running" {
t.Errorf("WaitingReboot = %q, want running", nodes[idxWaitingReboot].State)
}
if nodes[idxBooting].State != "pending" {
t.Errorf("Booting = %q, want pending", nodes[idxBooting].State)
}
}
// TestRenderPipelineString_IncludesSection asserts the orchestrator-
// published fragment carries the <section id=pipeline-N sse-swap=...
// hx-swap=outerHTML> wrapper. Without this, the first outerHTML swap
// would replace the section with a bare <div class=pipeline>, wiping
// out the sse-swap attribute and freezing every subsequent pipeline
// event until page reload.
func TestRenderPipelineString_IncludesSection(t *testing.T) {
run := &model.Run{ID: 42, State: model.StateSMART}
html := RenderPipelineString(run, seedStages())
for _, want := range []string{
`id="pipeline-42"`,
`sse-swap="pipeline-42"`,
`hx-swap="outerHTML"`,
`<h2>Pipeline</h2>`,
} {
if !strings.Contains(html, want) {
t.Errorf("RenderPipelineString missing %q in:\n%s", want, html)
}
}
}