Initial commit: full Phases 1-6 implementation

Post-repair hardware validation pipeline for Proxmox cluster hosts.
Go orchestrator + in-image agent + mkosi live image + bundled dnsmasq
PXE + SQLite + HTMX/SSE UI + notify registry + janitor + full docs.

internal/hold/hold.go

@@ -0,0 +1,65 @@
+// Package hold generates per-run ephemeral ed25519 keypairs for the
+// FailedHolding flow. When a run fails, the agent asks the orchestrator
+// for a pubkey, drops it into /root/.ssh/authorized_keys, and reports
+// its LAN IP. The orchestrator stores the private key next to the run's
+// artifacts and surfaces `ssh -i <path> root@<ip>` on the tile.
+package hold
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/pem"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// Keypair bundles the PEM-encoded private key and the
+// authorized_keys-style public key line.
+type Keypair struct {
+	PrivatePEM    []byte
+	AuthorizedKey string // "ssh-ed25519 AAAA... vetting-hold-N"
+}
+
+// Issue generates a new ed25519 keypair labelled for the given run.
+func Issue(runID int64) (*Keypair, error) {
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return nil, fmt.Errorf("generate ed25519: %w", err)
+	}
+	sshPub, err := ssh.NewPublicKey(pub)
+	if err != nil {
+		return nil, fmt.Errorf("ssh public key: %w", err)
+	}
+	blob := ssh.MarshalAuthorizedKey(sshPub) // "ssh-ed25519 AAAA...\n"
+	line := strings.TrimRight(string(blob), "\n")
+	if !strings.HasSuffix(line, fmt.Sprintf(" vetting-hold-%d", runID)) {
+		line += fmt.Sprintf(" vetting-hold-%d", runID)
+	}
+
+	block, err := ssh.MarshalPrivateKey(priv, fmt.Sprintf("vetting-hold-%d", runID))
+	if err != nil {
+		return nil, fmt.Errorf("marshal private key: %w", err)
+	}
+	return &Keypair{PrivatePEM: pem.EncodeToMemory(block), AuthorizedKey: line}, nil
+}
+
+// WritePrivateTo persists the PEM to the given path with 0600 perms
+// and returns the absolute path. The operator's shell reads this file
+// by path, so we keep it on disk per-run.
+func (kp *Keypair) WritePrivateTo(path string) (string, error) {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return "", err
+	}
+	if err := os.WriteFile(path, kp.PrivatePEM, 0o600); err != nil {
+		return "", fmt.Errorf("write hold key: %w", err)
+	}
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return path, nil
+	}
+	return abs, nil
+}

internal/hold/hold_test.go

@@ -0,0 +1,99 @@
+package hold
+
+import (
+	"bytes"
+	"crypto/ed25519"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// TestIssueRoundTrip checks that the private key we write is parseable
+// with the standard openssh library and that its derived public key
+// byte-for-byte matches the authorized_key line we handed the agent.
+// If this drifts — e.g. we swap from ed25519 to something else, or
+// mangle the comment — the operator's `ssh -i path root@ip` breaks
+// silently. The test is the only early-warning we have.
+func TestIssueRoundTrip(t *testing.T) {
+	kp, err := Issue(42)
+	if err != nil {
+		t.Fatalf("Issue: %v", err)
+	}
+
+	// Parse the private key back.
+	signer, err := ssh.ParsePrivateKey(kp.PrivatePEM)
+	if err != nil {
+		t.Fatalf("ParsePrivateKey: %v", err)
+	}
+
+	// The public derived from the signer must match the authorized_key line.
+	gotAuth := strings.TrimRight(string(ssh.MarshalAuthorizedKey(signer.PublicKey())), "\n")
+	wantAuth := kp.AuthorizedKey
+	// Authorized_keys comment is ours; compare just the type+b64 prefix.
+	gotParts := strings.SplitN(gotAuth, " ", 3)
+	wantParts := strings.SplitN(wantAuth, " ", 3)
+	if len(gotParts) < 2 || len(wantParts) < 2 {
+		t.Fatalf("unexpected authorized_key shape got=%q want=%q", gotAuth, wantAuth)
+	}
+	if gotParts[0] != wantParts[0] || gotParts[1] != wantParts[1] {
+		t.Fatalf("public key mismatch:\n  got  %s\n  want %s", gotAuth, wantAuth)
+	}
+	if !strings.Contains(wantAuth, "vetting-hold-42") {
+		t.Fatalf("authorized_key line missing run tag: %q", wantAuth)
+	}
+}
+
+// TestIssueKeysAreEd25519 pins the algorithm — anything other than
+// ed25519 would surprise operators who've been told their hold key is
+// ed25519 (and would change key-file sizes, path handling, etc.).
+func TestIssueKeysAreEd25519(t *testing.T) {
+	kp, err := Issue(1)
+	if err != nil {
+		t.Fatalf("Issue: %v", err)
+	}
+	signer, err := ssh.ParsePrivateKey(kp.PrivatePEM)
+	if err != nil {
+		t.Fatalf("ParsePrivateKey: %v", err)
+	}
+	if got := signer.PublicKey().Type(); got != ssh.KeyAlgoED25519 {
+		t.Fatalf("key algorithm: got %s, want ssh-ed25519", got)
+	}
+	// Paranoia: the Ed25519 public key underneath should be 32 bytes.
+	edPub, ok := signer.PublicKey().(ssh.CryptoPublicKey)
+	if !ok {
+		t.Fatalf("public key does not expose CryptoPublicKey")
+	}
+	raw, ok := edPub.CryptoPublicKey().(ed25519.PublicKey)
+	if !ok {
+		t.Fatalf("public key is not ed25519.PublicKey")
+	}
+	if len(raw) != ed25519.PublicKeySize {
+		t.Fatalf("ed25519 pubkey size = %d, want %d", len(raw), ed25519.PublicKeySize)
+	}
+}
+
+func TestWritePrivateToSetsPerms(t *testing.T) {
+	kp, err := Issue(7)
+	if err != nil {
+		t.Fatalf("Issue: %v", err)
+	}
+	dir := t.TempDir()
+	path := filepath.Join(dir, "nested", "hold.key")
+	abs, err := kp.WritePrivateTo(path)
+	if err != nil {
+		t.Fatalf("WritePrivateTo: %v", err)
+	}
+	if !filepath.IsAbs(abs) {
+		t.Fatalf("expected absolute path, got %q", abs)
+	}
+	buf, err := os.ReadFile(abs)
+	if err != nil {
+		t.Fatalf("ReadFile: %v", err)
+	}
+	if !bytes.Equal(buf, kp.PrivatePEM) {
+		t.Fatalf("on-disk bytes differ from in-memory PEM")
+	}
+}