Hammerhead/metrics.go

package hammerhead

import (
	"crypto/subtle"
	"fmt"
	"net/http"
	"sort"
	"strings"
	"time"
)

// metrics serves Prometheus text format at /metrics. It is off unless a token
// is configured (HAMMERHEAD_METRICS_TOKEN), and then requires it as a bearer token.
func (s *Server) metrics(w http.ResponseWriter, r *http.Request) {
	want := s.opts.MetricsToken
	if want == "" {
		http.NotFound(w, r)
		return
	}
	got := bearer(r)
	if subtle.ConstantTimeCompare([]byte(got), []byte(want)) != 1 {
		w.Header().Set("WWW-Authenticate", `Bearer realm="metrics"`)
		http.Error(w, "unauthorized", http.StatusUnauthorized)
		return
	}
	var b strings.Builder
	typ := map[string]bool{}
	line := func(name, kind, help string, value float64, labels ...string) {
		if !typ[name] {
			typ[name] = true
			fmt.Fprintf(&b, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, kind)
		}
		var l []string
		for i := 0; i+1 < len(labels); i += 2 {
			l = append(l, fmt.Sprintf(`%s="%s"`, labels[i], promEscape(labels[i+1])))
		}
		if len(l) > 0 {
			fmt.Fprintf(&b, "%s{%s} %v\n", name, strings.Join(l, ","), value)
		} else {
			fmt.Fprintf(&b, "%s %v\n", name, value)
		}
	}
	b2f := func(v bool) float64 {
		if v {
			return 1
		}
		return 0
	}
	now := time.Now()
	s.mu.Lock()
	orgName := func(id string) string {
		if o := s.findOrg(id); o != nil {
			return o.Name
		}
		return ""
	}
	for _, e := range s.st.Edges {
		line("hammerhead_edge_up", "gauge", "1 if the edge contacted Hammerhead recently", b2f(!e.LastSeen.IsZero() && now.Sub(e.LastSeen) <= s.offlineAfter()), "edge", e.Name)
	}
	for _, st := range s.st.Sites {
		if !st.Local {
			line("hammerhead_site_up", "gauge", "1 if the site agent contacted Hammerhead recently", b2f(!st.LastSeen.IsZero() && now.Sub(st.LastSeen) <= s.offlineAfter()), "site", st.Name, "org", orgName(st.OrgID))
		}
	}
	for _, m := range s.st.Mappings {
		org := orgName(m.OrgID)
		st, known := s.status[m.ID]
		line("hammerhead_port_listening", "gauge", "1 if the edge reports the port is open", b2f(m.Enabled && known && st.Error == ""), "port", m.Name, "org", org)
		for i, th := range st.Targets {
			if i < len(m.Targets) {
				line("hammerhead_target_healthy", "gauge", "1 if the target passes its health check", b2f(th.Healthy), "port", m.Name, "org", org, "target", fmt.Sprintf("%s:%d", m.Targets[i].Host, m.Targets[i].Port))
			}
		}
		if t := s.st.Totals[m.ID]; t != nil {
			line("hammerhead_port_connections_total", "counter", "Connections or requests served", float64(t.Conns), "port", m.Name, "org", org)
			line("hammerhead_port_bytes_in_total", "counter", "Bytes from clients to targets", float64(t.BytesIn), "port", m.Name, "org", org)
			line("hammerhead_port_bytes_out_total", "counter", "Bytes from targets to clients", float64(t.BytesOut), "port", m.Name, "org", org)
		}
	}
	for _, x := range s.st.MailEgress {
		org := orgName(x.OrgID)
		line("hammerhead_mail_relay_listening", "gauge", "1 if the mail relay is running", b2f(x.Enabled && s.mailStatus[x.ID] == ""), "relay", x.Name, "org", org)
		if c, ok := s.mailChecks[x.ID]; ok {
			line("hammerhead_mail_check_ok", "gauge", "1 if the deliverability check passes", b2f(c.PTRMatchesHelo), "relay", x.Name, "org", org, "check", "reverse_dns")
			line("hammerhead_mail_check_ok", "gauge", "1 if the deliverability check passes", b2f(c.HeloResolves), "relay", x.Name, "org", org, "check", "helo_resolves")
			line("hammerhead_mail_check_ok", "gauge", "1 if the deliverability check passes", b2f(c.Port25 == "open"), "relay", x.Name, "org", org, "check", "port25")
			for _, f := range s.mailEgressFindingsLocked(x, c) {
				if f.Domain != "" || f.ID == "blocklist" || f.ID == "outcomes" {
					line("hammerhead_mail_finding_ok", "gauge", "1 if a per-domain, blocklist or delivery check passes", b2f(f.Status == "pass"), "relay", x.Name, "org", org, "check", f.Label, "domain", f.Domain)
				}
			}
		}
	}
	firing := map[string]int{}
	for _, a := range s.alerts {
		if a.Fired {
			firing[orgName(a.OrgID)]++
		}
	}
	orgs := make([]string, 0, len(firing))
	for o := range firing {
		orgs = append(orgs, o)
	}
	sort.Strings(orgs)
	for _, o := range orgs {
		line("hammerhead_alerts_firing", "gauge", "Alerts currently firing", float64(firing[o]), "org", o)
	}
	line("hammerhead_users", "gauge", "Accounts", float64(len(s.st.Users)))
	line("hammerhead_orgs", "gauge", "Organizations", float64(len(s.st.Orgs)))
	s.mu.Unlock()
	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
	w.Write([]byte(b.String()))
}

func promEscape(s string) string {
	return strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`).Replace(s)
}