Post-repair hardware validation pipeline for Proxmox cluster hosts. Go orchestrator + in-image agent + mkosi live image + bundled dnsmasq PXE + SQLite + HTMX/SSE UI + notify registry + janitor + full docs.
This commit is contained in:
@@ -0,0 +1,268 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/smtp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stubNotifier records every Send call; it's the test harness for
|
||||
// Registry routing logic without hitting network.
|
||||
type stubNotifier struct {
|
||||
name string
|
||||
calls []Event
|
||||
mu sync.Mutex
|
||||
failOn Kind // if non-empty, returns an error when ev.Kind == failOn
|
||||
}
|
||||
|
||||
func (s *stubNotifier) Name() string { return s.name }
|
||||
|
||||
func (s *stubNotifier) Send(_ context.Context, ev Event) error {
|
||||
s.mu.Lock()
|
||||
s.calls = append(s.calls, ev)
|
||||
s.mu.Unlock()
|
||||
if s.failOn != "" && ev.Kind == s.failOn {
|
||||
return errFake("forced failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubNotifier) seen() []Event {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]Event(nil), s.calls...)
|
||||
}
|
||||
|
||||
type errFake string
|
||||
|
||||
func (e errFake) Error() string { return string(e) }
|
||||
|
||||
// awaitCalls spins until every stub has the expected count or the
|
||||
// deadline elapses — Dispatch uses goroutines so the test must wait.
|
||||
func awaitCalls(t *testing.T, want map[*stubNotifier]int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
ok := true
|
||||
for s, n := range want {
|
||||
if len(s.seen()) < n {
|
||||
ok = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
for s, n := range want {
|
||||
t.Errorf("notifier %q: got %d calls, want %d", s.name, len(s.seen()), n)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRoutesByKind(t *testing.T) {
|
||||
reg := NewRegistry(time.Second)
|
||||
a := &stubNotifier{name: "fails-only"}
|
||||
b := &stubNotifier{name: "everything"}
|
||||
reg.Register(a)
|
||||
reg.Register(b)
|
||||
reg.AddRoute(Route{MatchKind: []Kind{KindStageFailed}, Notifier: "fails-only"})
|
||||
reg.AddRoute(Route{Notifier: "everything"})
|
||||
|
||||
reg.Dispatch(Event{Kind: KindStageFailed, Severity: SeverityCritical})
|
||||
reg.Dispatch(Event{Kind: KindRunCompleted, Severity: SeverityInfo})
|
||||
|
||||
awaitCalls(t, map[*stubNotifier]int{a: 1, b: 2})
|
||||
if got := a.seen()[0].Kind; got != KindStageFailed {
|
||||
t.Fatalf("a got %q, want StageFailed", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRoutesBySeverity(t *testing.T) {
|
||||
reg := NewRegistry(time.Second)
|
||||
crit := &stubNotifier{name: "crit-only"}
|
||||
reg.Register(crit)
|
||||
reg.AddRoute(Route{MatchSeverity: []Severity{SeverityCritical}, Notifier: "crit-only"})
|
||||
|
||||
reg.Dispatch(Event{Kind: KindRunCompleted, Severity: SeverityInfo})
|
||||
reg.Dispatch(Event{Kind: KindHoldingOpened, Severity: SeverityCritical})
|
||||
|
||||
awaitCalls(t, map[*stubNotifier]int{crit: 1})
|
||||
if got := crit.seen()[0].Severity; got != SeverityCritical {
|
||||
t.Fatalf("got severity %q, want critical", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryDeduplicatesNotifiers(t *testing.T) {
|
||||
reg := NewRegistry(time.Second)
|
||||
n := &stubNotifier{name: "only"}
|
||||
reg.Register(n)
|
||||
// Two routes naming the same notifier — a single Dispatch should
|
||||
// fire once, not twice.
|
||||
reg.AddRoute(Route{MatchKind: []Kind{KindStageFailed}, Notifier: "only"})
|
||||
reg.AddRoute(Route{MatchSeverity: []Severity{SeverityCritical}, Notifier: "only"})
|
||||
|
||||
reg.Dispatch(Event{Kind: KindStageFailed, Severity: SeverityCritical})
|
||||
|
||||
awaitCalls(t, map[*stubNotifier]int{n: 1})
|
||||
}
|
||||
|
||||
func TestRegistryUnknownNotifierIsNoop(t *testing.T) {
|
||||
reg := NewRegistry(time.Second)
|
||||
reg.AddRoute(Route{Notifier: "does-not-exist"})
|
||||
// Should not panic or block.
|
||||
reg.Dispatch(Event{Kind: KindRunCompleted})
|
||||
}
|
||||
|
||||
func TestRegistryFailureDoesNotPoisonOthers(t *testing.T) {
|
||||
reg := NewRegistry(time.Second)
|
||||
bad := &stubNotifier{name: "bad", failOn: KindStageFailed}
|
||||
good := &stubNotifier{name: "good"}
|
||||
reg.Register(bad)
|
||||
reg.Register(good)
|
||||
reg.AddRoute(Route{Notifier: "bad"})
|
||||
reg.AddRoute(Route{Notifier: "good"})
|
||||
|
||||
reg.Dispatch(Event{Kind: KindStageFailed, Severity: SeverityCritical})
|
||||
|
||||
awaitCalls(t, map[*stubNotifier]int{bad: 1, good: 1})
|
||||
}
|
||||
|
||||
func TestNtfyNotifierPOSTsBodyAndHeaders(t *testing.T) {
|
||||
var captured *http.Request
|
||||
var body string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
captured = r
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
body = string(b)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
n := NewNtfy("n", srv.URL, "vetting")
|
||||
err := n.Send(context.Background(), Event{
|
||||
Kind: KindStageFailed,
|
||||
Severity: SeverityCritical,
|
||||
Title: "host-01 FAILED",
|
||||
Body: "SMART failed",
|
||||
URL: "https://vetting.example/reports/42",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("send: %v", err)
|
||||
}
|
||||
if captured.Method != http.MethodPost {
|
||||
t.Fatalf("method = %s, want POST", captured.Method)
|
||||
}
|
||||
if captured.URL.Path != "/vetting" {
|
||||
t.Fatalf("path = %s, want /vetting", captured.URL.Path)
|
||||
}
|
||||
if got := captured.Header.Get("X-Title"); got != "host-01 FAILED" {
|
||||
t.Fatalf("X-Title = %q", got)
|
||||
}
|
||||
if got := captured.Header.Get("X-Click"); got != "https://vetting.example/reports/42" {
|
||||
t.Fatalf("X-Click = %q", got)
|
||||
}
|
||||
if got := captured.Header.Get("X-Priority"); got != "5" {
|
||||
t.Fatalf("X-Priority = %q, want 5 for critical", got)
|
||||
}
|
||||
if body != "SMART failed" {
|
||||
t.Fatalf("body = %q, want %q", body, "SMART failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNtfyNotifierNon2xxErrors(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, "rate limited", http.StatusTooManyRequests)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
n := NewNtfy("n", srv.URL, "t")
|
||||
err := n.Send(context.Background(), Event{Kind: KindRunCompleted, Body: "x"})
|
||||
if err == nil || !strings.Contains(err.Error(), "429") {
|
||||
t.Fatalf("want 429 error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscordNotifierPOSTsEmbed(t *testing.T) {
|
||||
var body string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
body = string(b)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := NewDiscord("d", srv.URL)
|
||||
err := d.Send(context.Background(), Event{
|
||||
Kind: KindRunCompleted,
|
||||
Severity: SeverityInfo,
|
||||
Title: "host-01 passed",
|
||||
Body: "all green",
|
||||
URL: "https://vetting.example/reports/1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("send: %v", err)
|
||||
}
|
||||
// Body should be a JSON payload containing an embeds array with our
|
||||
// title/description/URL.
|
||||
for _, want := range []string{`"embeds"`, `"host-01 passed"`, `"all green"`, `reports/1`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("body missing %q: %s", want, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSMTPNotifierInvokesSendMail(t *testing.T) {
|
||||
var called int32
|
||||
var gotAddr, gotFrom string
|
||||
var gotTo []string
|
||||
var gotMsg []byte
|
||||
s := NewSMTP("s", "mail.example", 2525, "vetting@example", []string{"ops@example"})
|
||||
s.SendMailFn = func(addr string, _ smtp.Auth, from string, to []string, msg []byte) error {
|
||||
atomic.AddInt32(&called, 1)
|
||||
gotAddr, gotFrom, gotTo, gotMsg = addr, from, to, msg
|
||||
return nil
|
||||
}
|
||||
err := s.Send(context.Background(), Event{
|
||||
Kind: KindStageFailed, Title: "subj", Body: "failure body",
|
||||
URL: "https://vetting.example/reports/9",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("send: %v", err)
|
||||
}
|
||||
if atomic.LoadInt32(&called) != 1 {
|
||||
t.Fatal("SendMailFn not called")
|
||||
}
|
||||
if gotAddr != "mail.example:2525" {
|
||||
t.Fatalf("addr = %q", gotAddr)
|
||||
}
|
||||
if gotFrom != "vetting@example" {
|
||||
t.Fatalf("from = %q", gotFrom)
|
||||
}
|
||||
if len(gotTo) != 1 || gotTo[0] != "ops@example" {
|
||||
t.Fatalf("to = %v", gotTo)
|
||||
}
|
||||
s1 := string(gotMsg)
|
||||
for _, want := range []string{"Subject: subj", "failure body", "Link: https://vetting.example/reports/9"} {
|
||||
if !strings.Contains(s1, want) {
|
||||
t.Errorf("message missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSMTPNotifierRejectsIncompleteConfig(t *testing.T) {
|
||||
s := &SMTPNotifier{NameStr: "s"}
|
||||
if err := s.Send(context.Background(), Event{Kind: KindRunCompleted}); err == nil {
|
||||
t.Fatal("want error, got nil")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user