Hammerhead/dmarc.go

package hammerhead

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"sort"
	"strings"
	"time"

	"git.veillax.com/Veillax/reef/dmarc"
	"git.veillax.com/Veillax/reef/model"
)

// DMARC aggregate reports. Receivers that handle mail claiming to be from your
// domain mail a daily summary to the rua= address in its DMARC record. Those
// emails are uploaded here (the raw .xml, .gz, .zip or the whole .eml), parsed,
// and kept for 90 days; the dashboard shows how much of the mail sent as your
// domain passed, and which senders are failing.

const (
	maxDMARCUpload   = 8 << 20
	dmarcKeepDays    = 90
	dmarcWindowDays  = 30
	dmarcAlertMinMsg = 20  // messages from our own relays before their pass rate says anything
	dmarcAlertBelow  = 0.9 // alert when less of our own mail than this passes DMARC
)

// ---------- storage ----------

const dmarcSchema = `
CREATE TABLE IF NOT EXISTS dmarc_reports (
	org_id TEXT NOT NULL, domain TEXT NOT NULL, reporter TEXT NOT NULL, report_id TEXT NOT NULL,
	begin_ts INTEGER NOT NULL, end_ts INTEGER NOT NULL, added INTEGER NOT NULL, data TEXT NOT NULL,
	PRIMARY KEY (org_id, domain, reporter, report_id)
);
CREATE INDEX IF NOT EXISTS dmarc_end ON dmarc_reports (org_id, end_ts);
`

// addDMARC stores a report; it reports false if this exact report was already there.
func (d *DB) addDMARC(org string, r *dmarc.Report) (bool, error) {
	data, err := json.Marshal(r)
	if err != nil {
		return false, err
	}
	res, err := d.sql.Exec(`INSERT OR IGNORE INTO dmarc_reports (org_id, domain, reporter, report_id, begin_ts, end_ts, added, data) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
		org, r.Domain, strings.ToLower(r.Org), r.ReportID, ms(r.Begin), ms(r.End), ms(time.Now()), string(data))
	if err != nil {
		return false, err
	}
	n, _ := res.RowsAffected()
	return n > 0, nil
}

// dmarcReports returns an org's reports that ended after since.
func (d *DB) dmarcReports(org string, since time.Time) []*dmarc.Report {
	rows, err := d.sql.Query(`SELECT data FROM dmarc_reports WHERE org_id = ? AND end_ts > ? ORDER BY end_ts`, org, ms(since))
	if err != nil {
		return nil
	}
	defer rows.Close()
	var out []*dmarc.Report
	for rows.Next() {
		var data string
		var r dmarc.Report
		if rows.Scan(&data) == nil && json.Unmarshal([]byte(data), &r) == nil {
			out = append(out, &r)
		}
	}
	return out
}

func (d *DB) purgeDMARC(before time.Time) {
	d.sql.Exec(`DELETE FROM dmarc_reports WHERE end_ts < ?`, ms(before))
}

func (d *DB) deleteDMARCDomain(org, domain string) (int64, error) {
	var res sql.Result
	res, err := d.sql.Exec(`DELETE FROM dmarc_reports WHERE org_id = ? AND domain = ?`, org, domain)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}

// ---------- summary ----------

type dmarcSource struct {
	IP        string `json:"ip"`
	Messages  int    `json:"messages"`
	Passed    int    `json:"passed"` // messages that passed DMARC
	DKIMPass  int    `json:"dkimPass"`
	SPFPass   int    `json:"spfPass"`
	Ours      bool   `json:"ours"`      // the address is one of this org's mail egress IPs
	Rejected  int    `json:"rejected"`  // the receiver quarantined or rejected these
	Reporters int    `json:"reporters"` // how many receivers saw it
	Hint      string `json:"hint,omitempty"`
}

type dmarcDomain struct {
	Domain    string        `json:"domain"`
	Reports   int           `json:"reports"`
	Messages  int           `json:"messages"`
	Passed    int           `json:"passed"`
	OurMsgs   int           `json:"ourMessages"` // from our own relays
	OurPassed int           `json:"ourPassed"`
	Policy    string        `json:"policy"` // the latest published policy a receiver reported
	Pct       int           `json:"pct"`
	Reporters []string      `json:"reporters"`
	First     time.Time     `json:"first"`
	Last      time.Time     `json:"last"`
	Sources   []dmarcSource `json:"sources"`
	Advice    []string      `json:"advice,omitempty"`
}

// summarizeDMARC totals reports per domain. ours holds the org's egress source IPs.
func summarizeDMARC(reports []*dmarc.Report, ours map[string]bool) []dmarcDomain {
	type acc struct {
		d         *dmarcDomain
		src       map[string]*dmarcSource
		srcRep    map[string]map[string]bool
		reporters map[string]bool
		lastEnd   time.Time
	}
	by := map[string]*acc{}
	for _, r := range reports {
		a := by[r.Domain]
		if a == nil {
			a = &acc{d: &dmarcDomain{Domain: r.Domain, First: r.Begin}, src: map[string]*dmarcSource{}, srcRep: map[string]map[string]bool{}, reporters: map[string]bool{}}
			by[r.Domain] = a
		}
		d := a.d
		d.Reports++
		if r.Begin.Before(d.First) {
			d.First = r.Begin
		}
		if r.End.After(a.lastEnd) { // the newest report's policy is the one in force
			a.lastEnd, d.Last, d.Policy, d.Pct = r.End, r.End, r.Policy.P, r.Policy.Pct
		}
		a.reporters[r.Org] = true
		for _, rec := range r.Records {
			s := a.src[rec.SourceIP]
			if s == nil {
				s = &dmarcSource{IP: rec.SourceIP, Ours: ours[rec.SourceIP]}
				a.src[rec.SourceIP] = s
				a.srcRep[rec.SourceIP] = map[string]bool{}
			}
			a.srcRep[rec.SourceIP][r.Org] = true
			s.Messages += rec.Count
			d.Messages += rec.Count
			if rec.Passed() {
				s.Passed += rec.Count
				d.Passed += rec.Count
			}
			if rec.DKIM == "pass" {
				s.DKIMPass += rec.Count
			}
			if rec.SPF == "pass" {
				s.SPFPass += rec.Count
			}
			if rec.Disposition == "quarantine" || rec.Disposition == "reject" {
				s.Rejected += rec.Count
			}
			if s.Ours {
				d.OurMsgs += rec.Count
				if rec.Passed() {
					d.OurPassed += rec.Count
				}
			}
		}
	}
	var out []dmarcDomain
	for _, a := range by {
		d := a.d
		for ip, s := range a.src {
			s.Reporters = len(a.srcRep[ip])
			s.Hint = sourceHint(s, d.Domain)
			d.Sources = append(d.Sources, *s)
		}
		sort.Slice(d.Sources, func(i, j int) bool {
			fi, fj := d.Sources[i].Messages-d.Sources[i].Passed, d.Sources[j].Messages-d.Sources[j].Passed
			if fi != fj {
				return fi > fj
			}
			return d.Sources[i].Messages > d.Sources[j].Messages
		})
		for r := range a.reporters {
			d.Reporters = append(d.Reporters, r)
		}
		sort.Strings(d.Reporters)
		d.Advice = dmarcAdvice(d)
		out = append(out, *d)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Domain < out[j].Domain })
	return out
}

func sourceHint(s *dmarcSource, domain string) string {
	failed := s.Messages - s.Passed
	if failed == 0 {
		return ""
	}
	switch {
	case s.Ours && s.DKIMPass == 0 && s.SPFPass == 0:
		return "Your own relay's mail failed both SPF and DKIM alignment. Check that " + domain + " has a published DKIM key here and an SPF record authorizing " + s.IP + "."
	case s.Ours && s.DKIMPass == 0:
		return "Mail from your relay only passes through SPF. Create and publish a DKIM key for " + domain + " so it survives forwarding."
	case s.Ours:
		return "Some mail from your relay failed. It is probably mail signed or sent under a different domain than the From address."
	case s.Rejected >= failed:
		return "Receivers are already quarantining or rejecting this sender. If it isn't yours, your policy is working; if it is, add it to your SPF record and set up DKIM with it."
	default:
		return "A sender outside your relays is failing. If it's a service you use (a newsletter or helpdesk tool), add its SPF include and set up its DKIM; if you don't recognise it, someone may be spoofing your domain and a stricter policy would stop that."
	}
}

func dmarcAdvice(d *dmarcDomain) []string {
	var out []string
	failedOther := 0
	for _, s := range d.Sources {
		if !s.Ours {
			failedOther += s.Messages - s.Passed
		}
	}
	if d.OurMsgs >= dmarcAlertMinMsg && float64(d.OurPassed)/float64(d.OurMsgs) < dmarcAlertBelow {
		out = append(out, fmt.Sprintf("Only %d%% of the mail from your own relays passed DMARC. Fix this before tightening the policy.", 100*d.OurPassed/d.OurMsgs))
	}
	if d.Policy == "none" && d.Messages >= dmarcAlertMinMsg && failedOther == 0 && (d.OurMsgs == 0 || d.OurPassed == d.OurMsgs) {
		out = append(out, "Everything that sent as "+d.Domain+" passed. You can move the policy from p=none to p=quarantine, then p=reject.")
	}
	if d.Policy == "none" && failedOther > 0 {
		out = append(out, fmt.Sprintf("%d messages from other senders failed. Identify them below before you enforce a policy, or legitimate mail will be blocked.", failedOther))
	}
	if d.Policy != "none" && d.Pct > 0 && d.Pct < 100 {
		out = append(out, fmt.Sprintf("The policy applies to only %d%% of mail (pct=%d). Raise it to 100 once reports look clean.", d.Pct, d.Pct))
	}
	return out
}

// ---------- API ----------

func (s *Server) orgEgressIPsLocked(org string) map[string]bool {
	ips := map[string]bool{}
	for _, x := range s.st.MailEgress {
		if x.OrgID == org {
			ips[x.SourceIP] = true
		}
	}
	return ips
}

func (s *Server) dmarcDomainsLocked(org string) map[string]bool {
	doms := map[string]bool{}
	for _, k := range s.st.DKIM {
		if k.OrgID == org {
			doms[strings.ToLower(k.Domain)] = true
		}
	}
	return doms
}

// dmarcForOrg summarizes an org's recent reports. It reads the database without holding s.mu.
func (s *Server) dmarcForOrg(org string, ours map[string]bool) []dmarcDomain {
	return summarizeDMARC(s.db.dmarcReports(org, time.Now().AddDate(0, 0, -dmarcWindowDays)), ours)
}

func (s *Server) listDMARC(w http.ResponseWriter, r *http.Request, c *orgCtx) {
	s.mu.Lock()
	ours := s.orgEgressIPsLocked(c.Org.ID)
	s.mu.Unlock()
	out := s.dmarcForOrg(c.Org.ID, ours)
	if out == nil {
		out = []dmarcDomain{}
	}
	writeJSON(w, 200, out)
}

// uploadDMARC accepts one or more reports in the request body (a report, or a whole email with one attached).
func (s *Server) uploadDMARC(w http.ResponseWriter, r *http.Request, c *orgCtx) {
	body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxDMARCUpload))
	if err != nil {
		fail(w, 413, errors.New("that file is too large to be a DMARC report"))
		return
	}
	rep, err := dmarc.Parse(body)
	if err != nil {
		fail(w, 400, err)
		return
	}
	s.mu.Lock()
	known := s.dmarcDomainsLocked(c.Org.ID)[rep.Domain]
	s.mu.Unlock()
	if !known {
		failf(w, 400, "this report is about %s, which isn't a domain you have a DKIM key for. Create the key first, or check the report is yours.", rep.Domain)
		return
	}
	added, err := s.db.addDMARC(c.Org.ID, rep)
	if err != nil {
		fail(w, 500, err)
		return
	}
	if added {
		s.audit(c.Org.ID, c.Actor, "dmarc.upload", rep.Domain+" from "+rep.Org)
	}
	msgs := 0
	for _, rec := range rep.Records {
		msgs += rec.Count
	}
	writeJSON(w, 200, map[string]any{"domain": rep.Domain, "reporter": rep.Org, "messages": msgs, "duplicate": !added})
}

func (s *Server) deleteDMARC(w http.ResponseWriter, r *http.Request, c *orgCtx) {
	domain := strings.ToLower(r.PathValue("domain"))
	if _, err := s.db.deleteDMARCDomain(c.Org.ID, domain); err != nil {
		fail(w, 500, err)
		return
	}
	s.audit(c.Org.ID, c.Actor, "dmarc.delete", domain)
	ok(w)
}

// dmarcProblems derives the alerts from reports: the mail our own relays send
// as a domain is failing DMARC. Caller holds s.mu; the database is read inside,
// which takes no server locks.
func (s *Server) dmarcProblemsLocked(add func(key, org, kind, name, msg string)) {
	orgs := map[string]bool{}
	for _, k := range s.st.DKIM {
		orgs[k.OrgID] = true
	}
	for org := range orgs {
		for _, d := range s.dmarcForOrg(org, s.orgEgressIPsLocked(org)) {
			if d.OurMsgs >= dmarcAlertMinMsg && float64(d.OurPassed)/float64(d.OurMsgs) < dmarcAlertBelow {
				add("dmarcrate:"+org+":"+d.Domain, org, "mail", d.Domain,
					fmt.Sprintf("Only %d%% of the mail your relays sent as %s passed DMARC in the last %d days (%d of %d messages). Open Mail → DMARC reports for the failing senders.",
						100*d.OurPassed/d.OurMsgs, d.Domain, dmarcWindowDays, d.OurPassed, d.OurMsgs))
			}
		}
	}
}

// ---------- findings for the dashboard ----------

// mailEgressFindings lists the checklist for one egress. Caller holds s.mu.
func (s *Server) mailEgressFindingsLocked(x *model.MailEgress, chk model.MailCheck) []Finding {
	return deliverabilityFindings(x, chk, s.st.DKIM)
}