package hammerhead import ( "bytes" "encoding/json" "errors" "fmt" "log" "net/http" "net/url" "regexp" "sort" "strings" "time" "git.veillax.com/Veillax/reef/netguard" ) // Alerts: every few seconds Hammerhead derives the set of current problems from // what the edges and sites report (an edge that went quiet, a port that can't // bind, a target failing its health check, a certificate error, a mail relay // that isn't deliverable). A problem has to persist for the debounce time before // anyone is told, which keeps a restart or a blip from paging you. You get one // message when it starts and one when it clears, and optionally a reminder. // Active alerts are kept in the database so a restart doesn't repeat them. var topicName = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) const ( defaultDebounce = 2 * time.Minute defaultOffline = 90 * time.Second ) // AlertChannel is somewhere alerts are sent: an email address, a webhook, a // Sandbar server (which pages whoever is on call and tracks the alert), or a // Sandbar topic (a plain message for everyone subscribed to it). type AlertChannel struct { ID string `json:"id"` OrgID string `json:"orgId"` // "" = a server-level channel (for edges) Name string `json:"name"` Kind string `json:"kind"` // "email", "webhook", "sandbar" or "topic" Target string `json:"target"` // address(es), a webhook URL, or Sandbar's address (secret: URLs often carry tokens) Format string `json:"format"` // webhook body: "json", "slack", "discord" or "ntfy" Token string `json:"token"` // Sandbar: API key (secret, never sent back to the browser; optional for a topic) Topic string `json:"topic"` // topic: the topic to publish to Policy string `json:"policy"` // Sandbar: escalation policy id; blank sends the alert without paging anyone AllowPrivate bool `json:"allowPrivate"` Enabled bool `json:"enabled"` } type alertState struct { Key, OrgID, Kind, Name, Message string Since time.Time Fired bool Notified time.Time } type problem struct{ org, kind, name, msg string } type alertNote struct { Key string `json:"key"` // stable per problem, so a recovery can find its alert Status string `json:"status"` // "firing", "resolved" or "test" Org string `json:"org"` Kind string `json:"kind"` Name string `json:"name"` Message string `json:"message"` Since time.Time `json:"since"` Time time.Time `json:"time"` } func (s *Server) offlineAfter() time.Duration { if s.opts.OfflineAfter > 0 { return s.opts.OfflineAfter } return defaultOffline } // problemsLocked derives every current problem. Caller holds s.mu. func (s *Server) problemsLocked(now time.Time) map[string]problem { out := map[string]problem{} add := func(key, org, kind, name, msg string) { out[key] = problem{org, kind, name, msg} } quiet := func(t time.Time) (bool, string) { if t.IsZero() { return false, "" // never connected: not set up yet, nothing to alert about } d := now.Sub(t) return d > s.offlineAfter(), d.Round(time.Second).String() } edgeDown, siteDown := map[string]bool{}, map[string]bool{} for _, e := range s.st.Edges { if down, d := quiet(e.LastSeen); down { edgeDown[e.ID] = true msg := fmt.Sprintf("Edge %s has not contacted Hammerhead for %s", e.Name, d) add("edge:"+e.ID+"@", "", "edge", e.Name, msg) // server-level for _, org := range s.orgsUsingEdgeLocked(e.ID) { add("edge:"+e.ID+"@"+org, org, "edge", e.Name, msg) } } } for _, st := range s.st.Sites { if st.Local { continue } if down, d := quiet(st.LastSeen); down { siteDown[st.ID] = true add("site:"+st.ID, st.OrgID, "site", st.Name, fmt.Sprintf("Site %s has not contacted Hammerhead for %s", st.Name, d)) } } for _, m := range s.st.Mappings { if !m.Enabled || edgeDown[m.EdgeID] { continue // a dead edge's last report is stale: the edge alert already says it } st, known := s.status[m.ID] if !known { continue } if st.Error != "" { add("port:"+m.ID, m.OrgID, "port", m.Name, fmt.Sprintf("%s can't listen: %s", m.Name, st.Error)) } if strings.HasPrefix(st.Cert, "error") { add("cert:"+m.ID, m.OrgID, "certificate", m.Name, fmt.Sprintf("%s has a certificate problem: %s", m.Name, st.Cert)) } if m.HealthSeconds != -1 { for i, th := range st.Targets { if i >= len(m.Targets) || th.Healthy || siteDown[m.Targets[i].SiteID] { continue } t := m.Targets[i] add(fmt.Sprintf("target:%s:%d", m.ID, i), m.OrgID, "target", m.Name, fmt.Sprintf("%s: the target %s:%d isn't answering (%s)", m.Name, t.Host, t.Port, th.Detail)) } } } for _, x := range s.st.MailEgress { if !x.Enabled || edgeDown[x.EdgeID] || siteDown[x.SiteID] { continue } if msg := s.mailStatus[x.ID]; msg != "" { add("mail:"+x.ID, x.OrgID, "mail", x.Name, fmt.Sprintf("Mail relay %s can't run: %s", x.Name, msg)) } if chk, ok := s.mailChecks[x.ID]; ok { if !chk.PTRMatchesHelo { add("mailptr:"+x.ID, x.OrgID, "mail", x.Name, fmt.Sprintf("%s: reverse DNS of %s doesn't match %s, so receiving servers may reject its mail", x.Name, x.SourceIP, x.HeloName)) } if !chk.HeloResolves { add("mailhelo:"+x.ID, x.OrgID, "mail", x.Name, fmt.Sprintf("%s: %s doesn't resolve to %s", x.Name, x.HeloName, x.SourceIP)) } if chk.Port25 != "" && chk.Port25 != "open" { add("mail25:"+x.ID, x.OrgID, "mail", x.Name, fmt.Sprintf("%s: can't connect out on port 25 from %s", x.Name, x.SourceIP)) } } } for _, x := range s.st.MailEgress { if !x.Enabled || edgeDown[x.EdgeID] || siteDown[x.SiteID] { continue } chk, ok := s.mailChecks[x.ID] if !ok { continue } for _, f := range s.mailEgressFindingsLocked(x, chk) { // ptr, helo, port 25 and dkim have alerts of their own above if !f.alertWorthy() || f.ID == "ptr" || f.ID == "helo" || f.ID == "port25" || strings.HasPrefix(f.ID, "dkim:") { continue } msg := fmt.Sprintf("%s: %s: %s", x.Name, f.Label, f.Detail) if f.Hint != "" { msg += ". " + f.Hint } add("mailfind:"+x.ID+":"+f.ID, x.OrgID, "mail", x.Name, msg) } } s.dmarcProblemsLocked(add) for _, k := range s.st.DKIM { for _, x := range s.st.MailEgress { if x.OrgID != k.OrgID || !x.Enabled || edgeDown[x.EdgeID] { continue } if st := s.mailChecks[x.ID].DKIM[k.ID]; st == "missing" || st == "mismatch" { add("dkim:"+k.ID, k.OrgID, "mail", k.Domain, fmt.Sprintf("DKIM key %s._domainkey.%s is %s in DNS, so signed mail will fail verification", k.Selector, k.Domain, st)) break } } } return out } // judgeableLocked reports whether we currently have the data to decide an alert's // problem is gone. After a restart the live statuses are empty until edges and // sites report again; resolving then would announce a recovery that never happened. func (s *Server) judgeableLocked(st *alertState) bool { kind, rest, _ := strings.Cut(st.Key, ":") switch kind { case "port", "cert": if s.findMapping(rest) == nil { return true // the thing was deleted: nothing left to be wrong } _, known := s.status[rest] return known case "target": id, _, _ := strings.Cut(rest, ":") if s.findMapping(id) == nil { return true } _, known := s.status[id] return known case "mail": if s.findMailEgress(rest) == nil { return true } _, known := s.mailStatus[rest] return known case "mailfind": id, _, _ := strings.Cut(rest, ":") if s.findMailEgress(id) == nil { return true } _, known := s.mailChecks[id] return known case "mailptr", "mailhelo", "mail25": if s.findMailEgress(rest) == nil { return true } _, known := s.mailChecks[rest] return known case "dkim": return len(s.mailChecks) > 0 } return true // edges and sites carry their own last-seen time } func (s *Server) orgsUsingEdgeLocked(edgeID string) []string { seen := map[string]bool{} var out []string for _, m := range s.st.Mappings { if m.EdgeID == edgeID && !seen[m.OrgID] { seen[m.OrgID] = true out = append(out, m.OrgID) } } for _, x := range s.st.MailEgress { if x.EdgeID == edgeID && !seen[x.OrgID] { seen[x.OrgID] = true out = append(out, x.OrgID) } } return out } func (s *Server) debounceFor(orgID string) time.Duration { if s.opts.AlertDebounce > 0 { return s.opts.AlertDebounce } if o := s.findOrg(orgID); o != nil && o.AlertDebounceMin > 0 { return time.Duration(o.AlertDebounceMin) * time.Minute } return defaultDebounce } func (s *Server) repeatFor(orgID string) time.Duration { if o := s.findOrg(orgID); o != nil && o.AlertRepeatHours > 0 { return time.Duration(o.AlertRepeatHours) * time.Hour } return 0 } type delivery struct { note alertNote channels []AlertChannel } // evaluateAlerts moves alert state forward to now and sends whatever became due. func (s *Server) evaluateAlerts(now time.Time) { var out []delivery s.mu.Lock() probs := s.problemsLocked(now) orgName := func(id string) string { if o := s.findOrg(id); o != nil { return o.Name } return "server" } queue := func(st *alertState, status string) { var chans []AlertChannel for _, c := range s.st.Channels { if c.OrgID == st.OrgID && c.Enabled { chans = append(chans, *c) } } out = append(out, delivery{alertNote{Key: st.Key, Status: status, Org: orgName(st.OrgID), Kind: st.Kind, Name: st.Name, Message: st.Message, Since: st.Since, Time: now}, chans}) s.db.addAlertEvent(now, st.OrgID, st.Key, status, st.Name, st.Message) } keys := make([]string, 0, len(probs)) for k := range probs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { p := probs[k] st := s.alerts[k] if st == nil { st = &alertState{Key: k, OrgID: p.org, Kind: p.kind, Since: now} s.alerts[k] = st } st.Name, st.Message = p.name, p.msg switch { case !st.Fired && now.Sub(st.Since) >= s.debounceFor(p.org): st.Fired, st.Notified = true, now queue(st, "firing") case st.Fired && s.repeatFor(p.org) > 0 && now.Sub(st.Notified) >= s.repeatFor(p.org): st.Notified = now queue(st, "firing") } s.db.saveAlert(st) } for k, st := range s.alerts { if _, still := probs[k]; still { continue } if !s.judgeableLocked(st) { continue // e.g. just restarted: the agents haven't reported yet, so we can't say it cleared } if st.Fired { queue(st, "resolved") } delete(s.alerts, k) s.db.deleteAlert(k) } s.mu.Unlock() for _, d := range out { go s.deliver(d) } } func (s *Server) loadAlerts() { for _, a := range s.db.loadAlertRows() { a := a s.alerts[a.Key] = &a } } func (s *Server) alertLoop() { every := s.opts.AlertInterval if every <= 0 { every = 15 * time.Second } t := time.NewTicker(every) defer t.Stop() for { select { case <-s.stop: return case <-t.C: s.evaluateAlerts(time.Now()) } } } // ---------- delivery ---------- func (n alertNote) subject() string { switch n.Status { case "resolved": return "[Hammerhead] Resolved: " + n.Name case "test": return "[Hammerhead] Test alert" } return "[Hammerhead] Problem: " + n.Name } func (n alertNote) text() string { switch n.Status { case "resolved": return "Resolved: " + n.Message case "test": return "This is a test alert from Hammerhead. If you can read this, the channel works." } return "Problem: " + n.Message } func (s *Server) deliver(d delivery) { for _, ch := range d.channels { if err := s.send(ch, d.note); err != nil { log.Printf("alert channel %q: %v", ch.Name, err) } } } func (s *Server) send(ch AlertChannel, n alertNote) error { switch ch.Kind { case "email": var errs []error for _, to := range strings.Split(ch.Target, ",") { if to = strings.TrimSpace(to); to != "" { if err := s.Mailer(to, n.subject(), n.text()+"\n\nOrganization: "+n.Org+"\nSince: "+n.Since.Format(time.RFC1123)+"\n"); err != nil { errs = append(errs, err) } } } return errors.Join(errs...) case "sandbar": return s.sendSandbar(ch, n) case "topic": return s.sendTopic(ch, n) case "webhook": body, ctype, hdr := webhookBody(ch.Format, n) req, err := http.NewRequest("POST", ch.Target, bytes.NewReader(body)) if err != nil { return err } req.Header.Set("Content-Type", ctype) for k, v := range hdr { req.Header.Set(k, v) } resp, err := netguard.Client(ch.AllowPrivate).Do(req) if err != nil { return err } resp.Body.Close() if resp.StatusCode >= 300 { return fmt.Errorf("the webhook answered %s", resp.Status) } return nil } return errors.New("unknown channel kind") } // sendSandbar raises the alert on a Sandbar server, or resolves it there when the // problem clears. Sandbar de-duplicates by key, so a reminder updates the alert // that is already open instead of opening another. A test raises an info alert // with no policy (nobody is paged) and resolves it straight away. func (s *Server) sendSandbar(ch AlertChannel, n alertNote) error { post := func(body map[string]string) error { raw, _ := json.Marshal(body) req, err := http.NewRequest("POST", strings.TrimRight(ch.Target, "/")+"/api/v1/alerts", bytes.NewReader(raw)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+ch.Token) resp, err := netguard.Client(ch.AllowPrivate).Do(req) if err != nil { return err } defer resp.Body.Close() switch { case resp.StatusCode == 401 || resp.StatusCode == 403: return errors.New("Sandbar rejected the API key") case resp.StatusCode >= 300: return fmt.Errorf("Sandbar answered %s", resp.Status) } return nil } key := "hammerhead:" + n.Key switch n.Status { case "resolved": return post(map[string]string{"key": key, "status": "resolved", "source": "hammerhead"}) case "test": key = "hammerhead:test:" + ch.ID if err := post(map[string]string{"key": key, "source": "hammerhead", "summary": "Hammerhead test alert", "detail": n.text(), "severity": "info"}); err != nil { return err } return post(map[string]string{"key": key, "status": "resolved", "source": "hammerhead"}) } return post(map[string]string{"key": key, "source": "hammerhead", "summary": n.Name + ": " + n.Message, "detail": "Organization: " + n.Org + "\nSince: " + n.Since.Format(time.RFC1123), "severity": "critical", "policy": ch.Policy}) } // sendTopic publishes the alert as a message on a Sandbar topic (ntfy's request // works too), so everyone subscribed, including the ntfy phone apps, sees it. A // topic has no alert to close, so a recovery is a second, calmer message. func (s *Server) sendTopic(ch AlertChannel, n alertNote) error { prio, tags := "urgent", "rotating_light" if n.Status != "firing" { prio, tags = "default", "white_check_mark" } req, err := http.NewRequest("POST", strings.TrimRight(ch.Target, "/")+"/"+ch.Topic, strings.NewReader(n.text()+"\n\nOrganization: "+n.Org)) if err != nil { return err } req.Header.Set("Title", n.subject()) req.Header.Set("Priority", prio) req.Header.Set("Tags", tags) if ch.Token != "" { req.Header.Set("Authorization", "Bearer "+ch.Token) } resp, err := netguard.Client(ch.AllowPrivate).Do(req) if err != nil { return err } defer resp.Body.Close() switch { case resp.StatusCode == 401 || resp.StatusCode == 403: return errors.New("Sandbar rejected the API key") case resp.StatusCode >= 300: return fmt.Errorf("Sandbar answered %s", resp.Status) } return nil } func webhookBody(format string, n alertNote) (body []byte, ctype string, hdr map[string]string) { text := n.subject() + "\n" + n.text() switch format { case "slack": body, _ = json.Marshal(map[string]string{"text": text}) case "discord": body, _ = json.Marshal(map[string]string{"content": text}) case "ntfy": prio, tags := "high", "rotating_light" if n.Status != "firing" { prio, tags = "default", "white_check_mark" } return []byte(n.text()), "text/plain", map[string]string{"Title": n.subject(), "Priority": prio, "Tags": tags} default: body, _ = json.Marshal(n) } return body, "application/json", nil } // validateChannel normalises ch. allowPrivate may only be kept if the caller is trusted. func (s *Server) validateChannel(ch *AlertChannel, trusted bool) error { ch.Name = strings.TrimSpace(ch.Name) ch.Target = strings.TrimSpace(ch.Target) switch ch.Kind { case "email": for _, a := range strings.Split(ch.Target, ",") { if a = strings.TrimSpace(a); !strings.Contains(a, "@") || strings.ContainsAny(a, " \r\n<>") { return errors.New("enter one or more email addresses, separated by commas") } } ch.Format, ch.Token, ch.Policy, ch.Topic, ch.AllowPrivate = "", "", "", "", false case "sandbar": u, err := url.Parse(ch.Target) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return errors.New("enter Sandbar's address, such as https://sandbar.example.com") } if ch.Token = strings.TrimSpace(ch.Token); ch.Token == "" { return errors.New("enter an API key from Sandbar's Account page") } ch.Policy, ch.Format, ch.Topic = strings.TrimSpace(ch.Policy), "", "" if ch.AllowPrivate && !trusted { return errors.New("sending to private addresses (like a service on your home network) needs a server administrator to allow it for this organization") } case "topic": u, err := url.Parse(ch.Target) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return errors.New("enter Sandbar's address, such as https://sandbar.example.com") } if ch.Topic = strings.TrimSpace(ch.Topic); !topicName.MatchString(ch.Topic) { return errors.New("a topic is 1 to 64 letters, digits, - or _") } ch.Token, ch.Policy, ch.Format = strings.TrimSpace(ch.Token), "", "" if ch.AllowPrivate && !trusted { return errors.New("sending to private addresses (like a service on your home network) needs a server administrator to allow it for this organization") } case "webhook": u, err := url.Parse(ch.Target) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return errors.New("the webhook must be an http:// or https:// URL") } ch.Token, ch.Policy, ch.Topic = "", "", "" switch ch.Format { case "json", "slack", "discord", "ntfy": case "": ch.Format = "json" default: return errors.New("format must be json, slack, discord or ntfy") } if ch.AllowPrivate && !trusted { return errors.New("sending to private addresses (like a service on your home network) needs a server administrator to allow it for this organization") } default: return errors.New("a channel is an email, a webhook, a Sandbar server or a Sandbar topic") } if ch.Name == "" { ch.Name = ch.Kind } return nil }