Add the Hammerhead control plane

Dashboard and API, accounts (organizations, roles, invitations, 2FA, OIDC
single sign-on, API keys, revocable sessions), an encrypted SQLite database,
resource logins, alerts and /metrics, mail egress management with DKIM keys,
and the .env loader. Split out of the original single repository; depends on
Reef for the shared types. The end-to-end tests that run Hammerhead, Blacktip
and Bamboo together live in e2e/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

metrics.go

@@ -0,0 +1,112 @@
+package hammerhead
+
+import (
+	"crypto/subtle"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+)
+
+// metrics serves Prometheus text format at /metrics. It is off unless a token
+// is configured (HAMMERHEAD_METRICS_TOKEN), and then requires it as a bearer token.
+func (s *Server) metrics(w http.ResponseWriter, r *http.Request) {
+	want := s.opts.MetricsToken
+	if want == "" {
+		http.NotFound(w, r)
+		return
+	}
+	got := bearer(r)
+	if subtle.ConstantTimeCompare([]byte(got), []byte(want)) != 1 {
+		w.Header().Set("WWW-Authenticate", `Bearer realm="metrics"`)
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	var b strings.Builder
+	typ := map[string]bool{}
+	line := func(name, kind, help string, value float64, labels ...string) {
+		if !typ[name] {
+			typ[name] = true
+			fmt.Fprintf(&b, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, kind)
+		}
+		var l []string
+		for i := 0; i+1 < len(labels); i += 2 {
+			l = append(l, fmt.Sprintf(`%s="%s"`, labels[i], promEscape(labels[i+1])))
+		}
+		if len(l) > 0 {
+			fmt.Fprintf(&b, "%s{%s} %v\n", name, strings.Join(l, ","), value)
+		} else {
+			fmt.Fprintf(&b, "%s %v\n", name, value)
+		}
+	}
+	b2f := func(v bool) float64 {
+		if v {
+			return 1
+		}
+		return 0
+	}
+	now := time.Now()
+	s.mu.Lock()
+	orgName := func(id string) string {
+		if o := s.findOrg(id); o != nil {
+			return o.Name
+		}
+		return ""
+	}
+	for _, e := range s.st.Edges {
+		line("hammerhead_edge_up", "gauge", "1 if the edge contacted Hammerhead recently", b2f(!e.LastSeen.IsZero() && now.Sub(e.LastSeen) <= s.offlineAfter()), "edge", e.Name)
+	}
+	for _, st := range s.st.Sites {
+		if !st.Local {
+			line("hammerhead_site_up", "gauge", "1 if the site agent contacted Hammerhead recently", b2f(!st.LastSeen.IsZero() && now.Sub(st.LastSeen) <= s.offlineAfter()), "site", st.Name, "org", orgName(st.OrgID))
+		}
+	}
+	for _, m := range s.st.Mappings {
+		org := orgName(m.OrgID)
+		st, known := s.status[m.ID]
+		line("hammerhead_port_listening", "gauge", "1 if the edge reports the port is open", b2f(m.Enabled && known && st.Error == ""), "port", m.Name, "org", org)
+		for i, th := range st.Targets {
+			if i < len(m.Targets) {
+				line("hammerhead_target_healthy", "gauge", "1 if the target passes its health check", b2f(th.Healthy), "port", m.Name, "org", org, "target", fmt.Sprintf("%s:%d", m.Targets[i].Host, m.Targets[i].Port))
+			}
+		}
+		if t := s.st.Totals[m.ID]; t != nil {
+			line("hammerhead_port_connections_total", "counter", "Connections or requests served", float64(t.Conns), "port", m.Name, "org", org)
+			line("hammerhead_port_bytes_in_total", "counter", "Bytes from clients to targets", float64(t.BytesIn), "port", m.Name, "org", org)
+			line("hammerhead_port_bytes_out_total", "counter", "Bytes from targets to clients", float64(t.BytesOut), "port", m.Name, "org", org)
+		}
+	}
+	for _, x := range s.st.MailEgress {
+		org := orgName(x.OrgID)
+		line("hammerhead_mail_relay_listening", "gauge", "1 if the mail relay is running", b2f(x.Enabled && s.mailStatus[x.ID] == ""), "relay", x.Name, "org", org)
+		if c, ok := s.mailChecks[x.ID]; ok {
+			line("hammerhead_mail_check_ok", "gauge", "1 if the deliverability check passes", b2f(c.PTRMatchesHelo), "relay", x.Name, "org", org, "check", "reverse_dns")
+			line("hammerhead_mail_check_ok", "gauge", "1 if the deliverability check passes", b2f(c.HeloResolves), "relay", x.Name, "org", org, "check", "helo_resolves")
+			line("hammerhead_mail_check_ok", "gauge", "1 if the deliverability check passes", b2f(c.Port25 == "open"), "relay", x.Name, "org", org, "check", "port25")
+		}
+	}
+	firing := map[string]int{}
+	for _, a := range s.alerts {
+		if a.Fired {
+			firing[orgName(a.OrgID)]++
+		}
+	}
+	orgs := make([]string, 0, len(firing))
+	for o := range firing {
+		orgs = append(orgs, o)
+	}
+	sort.Strings(orgs)
+	for _, o := range orgs {
+		line("hammerhead_alerts_firing", "gauge", "Alerts currently firing", float64(firing[o]), "org", o)
+	}
+	line("hammerhead_users", "gauge", "Accounts", float64(len(s.st.Users)))
+	line("hammerhead_orgs", "gauge", "Organizations", float64(len(s.st.Orgs)))
+	s.mu.Unlock()
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+	w.Write([]byte(b.String()))
+}
+
+func promEscape(s string) string {
+	return strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`).Replace(s)
+}