package hammerhead import ( "database/sql" "encoding/json" "errors" "fmt" "io" "net/http" "sort" "strings" "time" "git.veillax.com/Veillax/reef/dmarc" "git.veillax.com/Veillax/reef/model" ) // DMARC aggregate reports. Receivers that handle mail claiming to be from your // domain mail a daily summary to the rua= address in its DMARC record. Those // emails are uploaded here (the raw .xml, .gz, .zip or the whole .eml), parsed, // and kept for 90 days; the dashboard shows how much of the mail sent as your // domain passed, and which senders are failing. const ( maxDMARCUpload = 8 << 20 dmarcKeepDays = 90 dmarcWindowDays = 30 dmarcAlertMinMsg = 20 // messages from our own relays before their pass rate says anything dmarcAlertBelow = 0.9 // alert when less of our own mail than this passes DMARC ) // ---------- storage ---------- const dmarcSchema = ` CREATE TABLE IF NOT EXISTS dmarc_reports ( org_id TEXT NOT NULL, domain TEXT NOT NULL, reporter TEXT NOT NULL, report_id TEXT NOT NULL, begin_ts INTEGER NOT NULL, end_ts INTEGER NOT NULL, added INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY (org_id, domain, reporter, report_id) ); CREATE INDEX IF NOT EXISTS dmarc_end ON dmarc_reports (org_id, end_ts); ` // addDMARC stores a report; it reports false if this exact report was already there. func (d *DB) addDMARC(org string, r *dmarc.Report) (bool, error) { data, err := json.Marshal(r) if err != nil { return false, err } res, err := d.sql.Exec(`INSERT OR IGNORE INTO dmarc_reports (org_id, domain, reporter, report_id, begin_ts, end_ts, added, data) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, org, r.Domain, strings.ToLower(r.Org), r.ReportID, ms(r.Begin), ms(r.End), ms(time.Now()), string(data)) if err != nil { return false, err } n, _ := res.RowsAffected() return n > 0, nil } // dmarcReports returns an org's reports that ended after since. func (d *DB) dmarcReports(org string, since time.Time) []*dmarc.Report { rows, err := d.sql.Query(`SELECT data FROM dmarc_reports WHERE org_id = ? AND end_ts > ? ORDER BY end_ts`, org, ms(since)) if err != nil { return nil } defer rows.Close() var out []*dmarc.Report for rows.Next() { var data string var r dmarc.Report if rows.Scan(&data) == nil && json.Unmarshal([]byte(data), &r) == nil { out = append(out, &r) } } return out } func (d *DB) purgeDMARC(before time.Time) { d.sql.Exec(`DELETE FROM dmarc_reports WHERE end_ts < ?`, ms(before)) } func (d *DB) deleteDMARCDomain(org, domain string) (int64, error) { var res sql.Result res, err := d.sql.Exec(`DELETE FROM dmarc_reports WHERE org_id = ? AND domain = ?`, org, domain) if err != nil { return 0, err } return res.RowsAffected() } // ---------- summary ---------- type dmarcSource struct { IP string `json:"ip"` Messages int `json:"messages"` Passed int `json:"passed"` // messages that passed DMARC DKIMPass int `json:"dkimPass"` SPFPass int `json:"spfPass"` Ours bool `json:"ours"` // the address is one of this org's mail egress IPs Rejected int `json:"rejected"` // the receiver quarantined or rejected these Reporters int `json:"reporters"` // how many receivers saw it Hint string `json:"hint,omitempty"` } type dmarcDomain struct { Domain string `json:"domain"` Reports int `json:"reports"` Messages int `json:"messages"` Passed int `json:"passed"` OurMsgs int `json:"ourMessages"` // from our own relays OurPassed int `json:"ourPassed"` Policy string `json:"policy"` // the latest published policy a receiver reported Pct int `json:"pct"` Reporters []string `json:"reporters"` First time.Time `json:"first"` Last time.Time `json:"last"` Sources []dmarcSource `json:"sources"` Advice []string `json:"advice,omitempty"` } // summarizeDMARC totals reports per domain. ours holds the org's egress source IPs. func summarizeDMARC(reports []*dmarc.Report, ours map[string]bool) []dmarcDomain { type acc struct { d *dmarcDomain src map[string]*dmarcSource srcRep map[string]map[string]bool reporters map[string]bool lastEnd time.Time } by := map[string]*acc{} for _, r := range reports { a := by[r.Domain] if a == nil { a = &acc{d: &dmarcDomain{Domain: r.Domain, First: r.Begin}, src: map[string]*dmarcSource{}, srcRep: map[string]map[string]bool{}, reporters: map[string]bool{}} by[r.Domain] = a } d := a.d d.Reports++ if r.Begin.Before(d.First) { d.First = r.Begin } if r.End.After(a.lastEnd) { // the newest report's policy is the one in force a.lastEnd, d.Last, d.Policy, d.Pct = r.End, r.End, r.Policy.P, r.Policy.Pct } a.reporters[r.Org] = true for _, rec := range r.Records { s := a.src[rec.SourceIP] if s == nil { s = &dmarcSource{IP: rec.SourceIP, Ours: ours[rec.SourceIP]} a.src[rec.SourceIP] = s a.srcRep[rec.SourceIP] = map[string]bool{} } a.srcRep[rec.SourceIP][r.Org] = true s.Messages += rec.Count d.Messages += rec.Count if rec.Passed() { s.Passed += rec.Count d.Passed += rec.Count } if rec.DKIM == "pass" { s.DKIMPass += rec.Count } if rec.SPF == "pass" { s.SPFPass += rec.Count } if rec.Disposition == "quarantine" || rec.Disposition == "reject" { s.Rejected += rec.Count } if s.Ours { d.OurMsgs += rec.Count if rec.Passed() { d.OurPassed += rec.Count } } } } var out []dmarcDomain for _, a := range by { d := a.d for ip, s := range a.src { s.Reporters = len(a.srcRep[ip]) s.Hint = sourceHint(s, d.Domain) d.Sources = append(d.Sources, *s) } sort.Slice(d.Sources, func(i, j int) bool { fi, fj := d.Sources[i].Messages-d.Sources[i].Passed, d.Sources[j].Messages-d.Sources[j].Passed if fi != fj { return fi > fj } return d.Sources[i].Messages > d.Sources[j].Messages }) for r := range a.reporters { d.Reporters = append(d.Reporters, r) } sort.Strings(d.Reporters) d.Advice = dmarcAdvice(d) out = append(out, *d) } sort.Slice(out, func(i, j int) bool { return out[i].Domain < out[j].Domain }) return out } func sourceHint(s *dmarcSource, domain string) string { failed := s.Messages - s.Passed if failed == 0 { return "" } switch { case s.Ours && s.DKIMPass == 0 && s.SPFPass == 0: return "Your own relay's mail failed both SPF and DKIM alignment. Check that " + domain + " has a published DKIM key here and an SPF record authorizing " + s.IP + "." case s.Ours && s.DKIMPass == 0: return "Mail from your relay only passes through SPF. Create and publish a DKIM key for " + domain + " so it survives forwarding." case s.Ours: return "Some mail from your relay failed. It is probably mail signed or sent under a different domain than the From address." case s.Rejected >= failed: return "Receivers are already quarantining or rejecting this sender. If it isn't yours, your policy is working; if it is, add it to your SPF record and set up DKIM with it." default: return "A sender outside your relays is failing. If it's a service you use (a newsletter or helpdesk tool), add its SPF include and set up its DKIM; if you don't recognise it, someone may be spoofing your domain and a stricter policy would stop that." } } func dmarcAdvice(d *dmarcDomain) []string { var out []string failedOther := 0 for _, s := range d.Sources { if !s.Ours { failedOther += s.Messages - s.Passed } } if d.OurMsgs >= dmarcAlertMinMsg && float64(d.OurPassed)/float64(d.OurMsgs) < dmarcAlertBelow { out = append(out, fmt.Sprintf("Only %d%% of the mail from your own relays passed DMARC. Fix this before tightening the policy.", 100*d.OurPassed/d.OurMsgs)) } if d.Policy == "none" && d.Messages >= dmarcAlertMinMsg && failedOther == 0 && (d.OurMsgs == 0 || d.OurPassed == d.OurMsgs) { out = append(out, "Everything that sent as "+d.Domain+" passed. You can move the policy from p=none to p=quarantine, then p=reject.") } if d.Policy == "none" && failedOther > 0 { out = append(out, fmt.Sprintf("%d messages from other senders failed. Identify them below before you enforce a policy, or legitimate mail will be blocked.", failedOther)) } if d.Policy != "none" && d.Pct > 0 && d.Pct < 100 { out = append(out, fmt.Sprintf("The policy applies to only %d%% of mail (pct=%d). Raise it to 100 once reports look clean.", d.Pct, d.Pct)) } return out } // ---------- API ---------- func (s *Server) orgEgressIPsLocked(org string) map[string]bool { ips := map[string]bool{} for _, x := range s.st.MailEgress { if x.OrgID == org { ips[x.SourceIP] = true } } return ips } func (s *Server) dmarcDomainsLocked(org string) map[string]bool { doms := map[string]bool{} for _, k := range s.st.DKIM { if k.OrgID == org { doms[strings.ToLower(k.Domain)] = true } } return doms } // dmarcForOrg summarizes an org's recent reports. It reads the database without holding s.mu. func (s *Server) dmarcForOrg(org string, ours map[string]bool) []dmarcDomain { return summarizeDMARC(s.db.dmarcReports(org, time.Now().AddDate(0, 0, -dmarcWindowDays)), ours) } func (s *Server) listDMARC(w http.ResponseWriter, r *http.Request, c *orgCtx) { s.mu.Lock() ours := s.orgEgressIPsLocked(c.Org.ID) s.mu.Unlock() out := s.dmarcForOrg(c.Org.ID, ours) if out == nil { out = []dmarcDomain{} } writeJSON(w, 200, out) } // uploadDMARC accepts one or more reports in the request body (a report, or a whole email with one attached). func (s *Server) uploadDMARC(w http.ResponseWriter, r *http.Request, c *orgCtx) { body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxDMARCUpload)) if err != nil { fail(w, 413, errors.New("that file is too large to be a DMARC report")) return } rep, err := dmarc.Parse(body) if err != nil { fail(w, 400, err) return } s.mu.Lock() known := s.dmarcDomainsLocked(c.Org.ID)[rep.Domain] s.mu.Unlock() if !known { failf(w, 400, "this report is about %s, which isn't a domain you have a DKIM key for. Create the key first, or check the report is yours.", rep.Domain) return } added, err := s.db.addDMARC(c.Org.ID, rep) if err != nil { fail(w, 500, err) return } if added { s.audit(c.Org.ID, c.Actor, "dmarc.upload", rep.Domain+" from "+rep.Org) } msgs := 0 for _, rec := range rep.Records { msgs += rec.Count } writeJSON(w, 200, map[string]any{"domain": rep.Domain, "reporter": rep.Org, "messages": msgs, "duplicate": !added}) } func (s *Server) deleteDMARC(w http.ResponseWriter, r *http.Request, c *orgCtx) { domain := strings.ToLower(r.PathValue("domain")) if _, err := s.db.deleteDMARCDomain(c.Org.ID, domain); err != nil { fail(w, 500, err) return } s.audit(c.Org.ID, c.Actor, "dmarc.delete", domain) ok(w) } // dmarcProblems derives the alerts from reports: the mail our own relays send // as a domain is failing DMARC. Caller holds s.mu; the database is read inside, // which takes no server locks. func (s *Server) dmarcProblemsLocked(add func(key, org, kind, name, msg string)) { orgs := map[string]bool{} for _, k := range s.st.DKIM { orgs[k.OrgID] = true } for org := range orgs { for _, d := range s.dmarcForOrg(org, s.orgEgressIPsLocked(org)) { if d.OurMsgs >= dmarcAlertMinMsg && float64(d.OurPassed)/float64(d.OurMsgs) < dmarcAlertBelow { add("dmarcrate:"+org+":"+d.Domain, org, "mail", d.Domain, fmt.Sprintf("Only %d%% of the mail your relays sent as %s passed DMARC in the last %d days (%d of %d messages). Open Mail → DMARC reports for the failing senders.", 100*d.OurPassed/d.OurMsgs, d.Domain, dmarcWindowDays, d.OurPassed, d.OurMsgs)) } } } } // ---------- findings for the dashboard ---------- // mailEgressFindings lists the checklist for one egress. Caller holds s.mu. func (s *Server) mailEgressFindingsLocked(x *model.MailEgress, chk model.MailCheck) []Finding { return deliverabilityFindings(x, chk, s.st.DKIM) }