23c689aa5b
Ships all five phases of the deep-profile overhaul together. Runs now carry a profile (quick/deep/soak); every profile walks the same 11-stage order — Inventory → Firmware → SpecValidate → SMART → CPUStress → Storage → Network → Burn → GPU → PSU → Reporting — with only per-stage durations and concurrency scaled. Phase 1: profiles.ProfileRegistry loaded from vetting.yaml; runs.profile column + CreateWithProfile; threshold table + evaluator seeded per-run from the shared vetting.thresholds block; breach flips result at /sensor + /result. Phase 2: upgraded CPUStress (stress-ng --cpu-method=all --verify + EDAC/MCE poll), Storage (fio --verify=md5 + SMART start/end delta), Network (sustained iperf + /proc/net/dev deltas) with per-profile knobs from Deps. Phase 3: Burn super-stage with goroutine fan-out for CPU + memory + fio + iperf, PSU rails sampled across the Burn window, SensorMux (2 s flush, 500-sample cap) to absorb backpressure. Phase 4: Firmware stage + firmware_snapshots table; probes dmidecode (BIOS), ipmitool (BMC), ethtool -i (NIC), nvme (sysfs + id-ctrl), lspci (HBA), /proc/cpuinfo (microcode). spec.DiffFirmware folds into SpecValidate with pin-by-identifier and fan-out-across-component matching; mismatches park the run in FailedHolding. Phase 5: profile radio on the host start form, profile chip on the run header, Firmware section in the HTML report, coverage artifact uploaded from CI, agent/tests/fakes/ scaffold with Deps.LookPath seam + stress_ng and dmidecode example fakes. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
229 lines
7.7 KiB
Go
229 lines
7.7 KiB
Go
package templates
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"vetting/internal/model"
|
|
)
|
|
|
|
// node indexes for the default pipeline layout: pre-stages (3) + stage
|
|
// rows (11) + terminal Completed (1) = 15 nodes.
|
|
const (
|
|
idxQueued = 0
|
|
idxWaitingReboot = 1
|
|
idxBooting = 2
|
|
idxInventory = 3
|
|
idxFirmware = 4
|
|
idxSpecValidate = 5
|
|
idxSMART = 6
|
|
idxCPUStress = 7
|
|
idxStorage = 8
|
|
idxNetwork = 9
|
|
idxBurn = 10
|
|
idxGPU = 11
|
|
idxPSU = 12
|
|
idxReporting = 13
|
|
idxCompleted = 14
|
|
)
|
|
|
|
// seedStages returns a fresh all-pending stage slice in the canonical order.
|
|
func seedStages() []model.Stage {
|
|
names := []string{"Inventory", "Firmware", "SpecValidate", "SMART", "CPUStress", "Storage", "Network", "Burn", "GPU", "PSU", "Reporting"}
|
|
out := make([]model.Stage, len(names))
|
|
for i, n := range names {
|
|
out[i] = model.Stage{Name: n, Ordinal: i, State: model.StagePending}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestBuildPipeline_NoRun(t *testing.T) {
|
|
nodes := BuildPipeline(nil, nil)
|
|
// Ghost pipeline: 3 pre-stages + 10 stage ghosts + 1 terminal = 14
|
|
// nodes, all pending.
|
|
if len(nodes) != 15 {
|
|
t.Fatalf("len = %d, want 15", len(nodes))
|
|
}
|
|
for i, n := range nodes {
|
|
if n.State != "pending" {
|
|
t.Errorf("node %d (%s) state = %q, want pending", i, n.Name, n.State)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestBuildPipeline_GhostStagesBeforeClaim models the real WaitingReboot
|
|
// case: the run exists but agent hasn't called /claim yet, so there are
|
|
// no stage rows. Pipeline must still render all 9 stage nodes as ghosts
|
|
// so the operator sees the full timeline ahead of them.
|
|
func TestBuildPipeline_GhostStagesBeforeClaim(t *testing.T) {
|
|
run := &model.Run{State: model.StateWaitingReboot}
|
|
nodes := BuildPipeline(run, nil)
|
|
if len(nodes) != 15 {
|
|
t.Fatalf("len = %d, want 15", len(nodes))
|
|
}
|
|
if nodes[idxQueued].State != "passed" {
|
|
t.Errorf("Queued = %q, want passed", nodes[idxQueued].State)
|
|
}
|
|
if nodes[idxWaitingReboot].State != "running" {
|
|
t.Errorf("WaitingReboot = %q, want running", nodes[idxWaitingReboot].State)
|
|
}
|
|
// All 11 stage ghosts must be pending — nothing has started yet.
|
|
for i := idxInventory; i <= idxReporting; i++ {
|
|
if nodes[i].State != "pending" {
|
|
t.Errorf("%s (ghost) = %q, want pending", nodes[i].Name, nodes[i].State)
|
|
}
|
|
}
|
|
if nodes[idxCompleted].State != "pending" {
|
|
t.Errorf("Completed = %q, want pending", nodes[idxCompleted].State)
|
|
}
|
|
}
|
|
|
|
// TestBuildPipeline_GhostStagesDuringStage models the in-flight case
|
|
// with only some stage rows seeded: later stages must still appear as
|
|
// pending ghosts rather than silently disappearing.
|
|
func TestBuildPipeline_GhostStagesDuringStage(t *testing.T) {
|
|
run := &model.Run{State: model.StateSMART}
|
|
// Only Inventory + Firmware + SpecValidate seeded; SMART onwards are ghosts.
|
|
stages := []model.Stage{
|
|
{Name: "Inventory", Ordinal: 0, State: model.StagePassed},
|
|
{Name: "Firmware", Ordinal: 1, State: model.StagePassed},
|
|
{Name: "SpecValidate", Ordinal: 2, State: model.StagePassed},
|
|
}
|
|
nodes := BuildPipeline(run, stages)
|
|
if len(nodes) != 15 {
|
|
t.Fatalf("len = %d, want 15", len(nodes))
|
|
}
|
|
if nodes[idxSMART].State != "running" {
|
|
t.Errorf("SMART (ghost) = %q, want running", nodes[idxSMART].State)
|
|
}
|
|
for _, i := range []int{idxCPUStress, idxStorage, idxNetwork, idxBurn, idxGPU, idxPSU, idxReporting} {
|
|
if nodes[i].State != "pending" {
|
|
t.Errorf("%s (ghost) = %q, want pending", nodes[i].Name, nodes[i].State)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBuildPipeline_Running(t *testing.T) {
|
|
run := &model.Run{State: model.StateSMART}
|
|
stages := seedStages()
|
|
stages[0].State = model.StagePassed // Inventory
|
|
stages[1].State = model.StagePassed // Firmware
|
|
stages[2].State = model.StagePassed // SpecValidate
|
|
stages[3].State = model.StageRunning // SMART
|
|
nodes := BuildPipeline(run, stages)
|
|
if len(nodes) != 15 {
|
|
t.Fatalf("len = %d, want 15", len(nodes))
|
|
}
|
|
// Pre-stages are all past for a run that has reached SMART.
|
|
for i := idxQueued; i <= idxBooting; i++ {
|
|
if nodes[i].State != "passed" {
|
|
t.Errorf("prestage %s = %q, want passed", nodes[i].Name, nodes[i].State)
|
|
}
|
|
}
|
|
if nodes[idxInventory].State != "passed" {
|
|
t.Errorf("Inventory = %q, want passed", nodes[idxInventory].State)
|
|
}
|
|
if nodes[idxSpecValidate].State != "passed" {
|
|
t.Errorf("SpecValidate = %q, want passed", nodes[idxSpecValidate].State)
|
|
}
|
|
if nodes[idxSMART].State != "running" {
|
|
t.Errorf("SMART = %q, want running", nodes[idxSMART].State)
|
|
}
|
|
if nodes[idxCPUStress].State != "pending" {
|
|
t.Errorf("CPUStress = %q, want pending", nodes[idxCPUStress].State)
|
|
}
|
|
if nodes[idxCompleted].State != "pending" {
|
|
t.Errorf("Completed = %q, want pending", nodes[idxCompleted].State)
|
|
}
|
|
}
|
|
|
|
func TestBuildPipeline_Failed(t *testing.T) {
|
|
run := &model.Run{State: model.StateFailedHolding, FailedStage: "Storage"}
|
|
stages := seedStages()
|
|
for i := 0; i <= 4; i++ {
|
|
stages[i].State = model.StagePassed
|
|
}
|
|
stages[5].State = model.StageFailed // Storage
|
|
nodes := BuildPipeline(run, stages)
|
|
// Pre-stages are past a run that reached Storage.
|
|
for i := idxQueued; i <= idxBooting; i++ {
|
|
if nodes[i].State != "passed" {
|
|
t.Errorf("prestage %s = %q, want passed", nodes[i].Name, nodes[i].State)
|
|
}
|
|
}
|
|
if nodes[idxStorage].State != "failed" {
|
|
t.Errorf("Storage = %q, want failed", nodes[idxStorage].State)
|
|
}
|
|
for _, i := range []int{idxNetwork, idxBurn, idxGPU, idxPSU, idxReporting} {
|
|
if nodes[i].State != "skipped" {
|
|
t.Errorf("%s = %q, want skipped", nodes[i].Name, nodes[i].State)
|
|
}
|
|
}
|
|
if nodes[idxCompleted].State != "pending" {
|
|
t.Errorf("Completed = %q, want pending on failure", nodes[idxCompleted].State)
|
|
}
|
|
}
|
|
|
|
func TestBuildPipeline_Completed(t *testing.T) {
|
|
run := &model.Run{State: model.StateCompleted}
|
|
stages := seedStages()
|
|
for i := range stages {
|
|
stages[i].State = model.StagePassed
|
|
}
|
|
nodes := BuildPipeline(run, stages)
|
|
for i, n := range nodes {
|
|
if n.State != "passed" {
|
|
t.Errorf("node %d (%s) state = %q, want passed", i, n.Name, n.State)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBuildPipeline_QueuedNow(t *testing.T) {
|
|
run := &model.Run{State: model.StateQueued}
|
|
nodes := BuildPipeline(run, seedStages())
|
|
if nodes[idxQueued].State != "running" {
|
|
t.Errorf("Queued = %q, want running", nodes[idxQueued].State)
|
|
}
|
|
if nodes[idxWaitingReboot].State != "pending" {
|
|
t.Errorf("WaitingReboot = %q, want pending", nodes[idxWaitingReboot].State)
|
|
}
|
|
}
|
|
|
|
// TestBuildPipeline_PreStageRunning_WaitingReboot confirms the pre-stage
|
|
// node for WaitingReboot lights up while the run sits there — the new
|
|
// happy-path state must map onto its pipeline slot.
|
|
func TestBuildPipeline_PreStageRunning_WaitingReboot(t *testing.T) {
|
|
run := &model.Run{State: model.StateWaitingReboot}
|
|
nodes := BuildPipeline(run, seedStages())
|
|
if nodes[idxQueued].State != "passed" {
|
|
t.Errorf("Queued = %q, want passed", nodes[idxQueued].State)
|
|
}
|
|
if nodes[idxWaitingReboot].State != "running" {
|
|
t.Errorf("WaitingReboot = %q, want running", nodes[idxWaitingReboot].State)
|
|
}
|
|
if nodes[idxBooting].State != "pending" {
|
|
t.Errorf("Booting = %q, want pending", nodes[idxBooting].State)
|
|
}
|
|
}
|
|
|
|
// TestRenderPipelineString_IncludesSection asserts the orchestrator-
|
|
// published fragment carries the <section id=pipeline-N sse-swap=...
|
|
// hx-swap=outerHTML> wrapper. Without this, the first outerHTML swap
|
|
// would replace the section with a bare <div class=pipeline>, wiping
|
|
// out the sse-swap attribute and freezing every subsequent pipeline
|
|
// event until page reload.
|
|
func TestRenderPipelineString_IncludesSection(t *testing.T) {
|
|
run := &model.Run{ID: 42, State: model.StateSMART}
|
|
html := RenderPipelineString(run, seedStages())
|
|
for _, want := range []string{
|
|
`id="pipeline-42"`,
|
|
`sse-swap="pipeline-42"`,
|
|
`hx-swap="outerHTML"`,
|
|
`<h2>Pipeline</h2>`,
|
|
} {
|
|
if !strings.Contains(html, want) {
|
|
t.Errorf("RenderPipelineString missing %q in:\n%s", want, html)
|
|
}
|
|
}
|
|
}
|