package hammerhead import ( "crypto/subtle" "fmt" "net/http" "sort" "strings" "time" ) // metrics serves Prometheus text format at /metrics. It is off unless a token // is configured (HAMMERHEAD_METRICS_TOKEN), and then requires it as a bearer token. func (s *Server) metrics(w http.ResponseWriter, r *http.Request) { want := s.opts.MetricsToken if want == "" { http.NotFound(w, r) return } got := bearer(r) if subtle.ConstantTimeCompare([]byte(got), []byte(want)) != 1 { w.Header().Set("WWW-Authenticate", `Bearer realm="metrics"`) http.Error(w, "unauthorized", http.StatusUnauthorized) return } var b strings.Builder typ := map[string]bool{} line := func(name, kind, help string, value float64, labels ...string) { if !typ[name] { typ[name] = true fmt.Fprintf(&b, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, kind) } var l []string for i := 0; i+1 < len(labels); i += 2 { l = append(l, fmt.Sprintf(`%s="%s"`, labels[i], promEscape(labels[i+1]))) } if len(l) > 0 { fmt.Fprintf(&b, "%s{%s} %v\n", name, strings.Join(l, ","), value) } else { fmt.Fprintf(&b, "%s %v\n", name, value) } } b2f := func(v bool) float64 { if v { return 1 } return 0 } now := time.Now() s.mu.Lock() orgName := func(id string) string { if o := s.findOrg(id); o != nil { return o.Name } return "" } for _, e := range s.st.Edges { line("hammerhead_edge_up", "gauge", "1 if the edge contacted Hammerhead recently", b2f(!e.LastSeen.IsZero() && now.Sub(e.LastSeen) <= s.offlineAfter()), "edge", e.Name) } for _, st := range s.st.Sites { if !st.Local { line("hammerhead_site_up", "gauge", "1 if the site agent contacted Hammerhead recently", b2f(!st.LastSeen.IsZero() && now.Sub(st.LastSeen) <= s.offlineAfter()), "site", st.Name, "org", orgName(st.OrgID)) } } for _, m := range s.st.Mappings { org := orgName(m.OrgID) st, known := s.status[m.ID] line("hammerhead_port_listening", "gauge", "1 if the edge reports the port is open", b2f(m.Enabled && known && st.Error == ""), "port", m.Name, "org", org) for i, th := range st.Targets { if i < len(m.Targets) { line("hammerhead_target_healthy", "gauge", "1 if the target passes its health check", b2f(th.Healthy), "port", m.Name, "org", org, "target", fmt.Sprintf("%s:%d", m.Targets[i].Host, m.Targets[i].Port)) } } if t := s.st.Totals[m.ID]; t != nil { line("hammerhead_port_connections_total", "counter", "Connections or requests served", float64(t.Conns), "port", m.Name, "org", org) line("hammerhead_port_bytes_in_total", "counter", "Bytes from clients to targets", float64(t.BytesIn), "port", m.Name, "org", org) line("hammerhead_port_bytes_out_total", "counter", "Bytes from targets to clients", float64(t.BytesOut), "port", m.Name, "org", org) } } for _, x := range s.st.MailEgress { org := orgName(x.OrgID) line("hammerhead_mail_relay_listening", "gauge", "1 if the mail relay is running", b2f(x.Enabled && s.mailStatus[x.ID] == ""), "relay", x.Name, "org", org) if c, ok := s.mailChecks[x.ID]; ok { line("hammerhead_mail_check_ok", "gauge", "1 if the deliverability check passes", b2f(c.PTRMatchesHelo), "relay", x.Name, "org", org, "check", "reverse_dns") line("hammerhead_mail_check_ok", "gauge", "1 if the deliverability check passes", b2f(c.HeloResolves), "relay", x.Name, "org", org, "check", "helo_resolves") line("hammerhead_mail_check_ok", "gauge", "1 if the deliverability check passes", b2f(c.Port25 == "open"), "relay", x.Name, "org", org, "check", "port25") for _, f := range s.mailEgressFindingsLocked(x, c) { if f.Domain != "" || f.ID == "blocklist" || f.ID == "outcomes" { line("hammerhead_mail_finding_ok", "gauge", "1 if a per-domain, blocklist or delivery check passes", b2f(f.Status == "pass"), "relay", x.Name, "org", org, "check", f.Label, "domain", f.Domain) } } } } firing := map[string]int{} for _, a := range s.alerts { if a.Fired { firing[orgName(a.OrgID)]++ } } orgs := make([]string, 0, len(firing)) for o := range firing { orgs = append(orgs, o) } sort.Strings(orgs) for _, o := range orgs { line("hammerhead_alerts_firing", "gauge", "Alerts currently firing", float64(firing[o]), "org", o) } line("hammerhead_users", "gauge", "Accounts", float64(len(s.st.Users))) line("hammerhead_orgs", "gauge", "Organizations", float64(len(s.st.Orgs))) s.mu.Unlock() w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") w.Write([]byte(b.String())) } func promEscape(s string) string { return strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`).Replace(s) }