Parse DMARC aggregate reports; model deliverability findings and delivery outcomes

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

dmarc/dmarc.go

@@ -0,0 +1,274 @@
+// Package dmarc parses DMARC aggregate (rua) reports (RFC 7489 appendix A).
+// Receivers send them as XML, usually gzip or zip compressed, usually as an
+// email attachment; Parse accepts any of those forms.
+package dmarc
+
+import (
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"encoding/xml"
+	"errors"
+	"io"
+	"mime"
+	"mime/multipart"
+	"mime/quotedprintable"
+	"net/mail"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// maxXML caps a decompressed report; aggregate reports are small, so a larger one is a zip bomb.
+const maxXML = 32 << 20
+
+// Report is one receiver's summary of the mail it saw claiming to be from Domain.
+type Report struct {
+	Org      string    `json:"org"` // the receiver that sent the report
+	Email    string    `json:"email"`
+	ReportID string    `json:"reportId"`
+	Begin    time.Time `json:"begin"`
+	End      time.Time `json:"end"`
+	Domain   string    `json:"domain"`
+	Policy   Policy    `json:"policy"`
+	Records  []Record  `json:"records"`
+}
+
+// Policy is the DMARC policy the receiver applied.
+type Policy struct {
+	P     string `json:"p"`   // none, quarantine or reject
+	SP    string `json:"sp"`  // for subdomains
+	Pct   int    `json:"pct"` // percent of mail the policy applies to
+	ADKIM string `json:"adkim"`
+	ASPF  string `json:"aspf"`
+}
+
+// Record is one row: a sending IP and what happened to its mail.
+type Record struct {
+	SourceIP     string       `json:"sourceIp"`
+	Count        int          `json:"count"`
+	Disposition  string       `json:"disposition"` // what the receiver did: none, quarantine, reject
+	DKIM         string       `json:"dkim"`        // aligned DKIM result: pass or fail
+	SPF          string       `json:"spf"`         // aligned SPF result: pass or fail
+	HeaderFrom   string       `json:"headerFrom"`
+	EnvelopeFrom string       `json:"envelopeFrom,omitempty"`
+	DKIMAuth     []AuthResult `json:"dkimAuth,omitempty"`
+	SPFAuth      []AuthResult `json:"spfAuth,omitempty"`
+}
+
+// AuthResult is a raw DKIM or SPF verdict, before alignment with the From domain.
+type AuthResult struct {
+	Domain   string `json:"domain"`
+	Selector string `json:"selector,omitempty"`
+	Result   string `json:"result"`
+}
+
+// Passed reports whether the row's mail passed DMARC: either aligned check passing is enough.
+func (r Record) Passed() bool { return r.DKIM == "pass" || r.SPF == "pass" }
+
+type xmlFeedback struct {
+	Meta struct {
+		Org      string `xml:"org_name"`
+		Email    string `xml:"email"`
+		ReportID string `xml:"report_id"`
+		Range    struct {
+			Begin int64 `xml:"begin"`
+			End   int64 `xml:"end"`
+		} `xml:"date_range"`
+	} `xml:"report_metadata"`
+	Policy struct {
+		Domain string `xml:"domain"`
+		ADKIM  string `xml:"adkim"`
+		ASPF   string `xml:"aspf"`
+		P      string `xml:"p"`
+		SP     string `xml:"sp"`
+		Pct    string `xml:"pct"`
+	} `xml:"policy_published"`
+	Records []struct {
+		Row struct {
+			SourceIP string `xml:"source_ip"`
+			Count    int    `xml:"count"`
+			Eval     struct {
+				Disposition string `xml:"disposition"`
+				DKIM        string `xml:"dkim"`
+				SPF         string `xml:"spf"`
+			} `xml:"policy_evaluated"`
+		} `xml:"row"`
+		Identifiers struct {
+			HeaderFrom   string `xml:"header_from"`
+			EnvelopeFrom string `xml:"envelope_from"`
+		} `xml:"identifiers"`
+		Auth struct {
+			DKIM []struct {
+				Domain   string `xml:"domain"`
+				Selector string `xml:"selector"`
+				Result   string `xml:"result"`
+			} `xml:"dkim"`
+			SPF []struct {
+				Domain string `xml:"domain"`
+				Result string `xml:"result"`
+			} `xml:"spf"`
+		} `xml:"auth_results"`
+	} `xml:"record"`
+}
+
+// Parse reads a report from raw XML, gzip, zip, or a whole email carrying one of those.
+func Parse(data []byte) (*Report, error) {
+	xmlData, err := unwrap(data, 0)
+	if err != nil {
+		return nil, err
+	}
+	var f xmlFeedback
+	if err := xml.Unmarshal(xmlData, &f); err != nil {
+		return nil, errors.New("not a DMARC report: " + err.Error())
+	}
+	if f.Policy.Domain == "" {
+		return nil, errors.New("not a DMARC report: no published policy domain")
+	}
+	if strings.TrimSpace(f.Meta.ReportID) == "" {
+		return nil, errors.New("not a DMARC report: no report_id")
+	}
+	pct := 100
+	if n, err := strconv.Atoi(strings.TrimSpace(f.Policy.Pct)); err == nil && n >= 0 && n <= 100 {
+		pct = n
+	}
+	r := &Report{
+		Org: strings.TrimSpace(f.Meta.Org), Email: strings.TrimSpace(f.Meta.Email), ReportID: strings.TrimSpace(f.Meta.ReportID),
+		Begin:  time.Unix(f.Meta.Range.Begin, 0).UTC(),
+		End:    time.Unix(f.Meta.Range.End, 0).UTC(),
+		Domain: lower(f.Policy.Domain),
+		Policy: Policy{P: lower(f.Policy.P), SP: lower(f.Policy.SP), Pct: pct, ADKIM: lower(f.Policy.ADKIM), ASPF: lower(f.Policy.ASPF)},
+	}
+	for _, x := range f.Records {
+		rec := Record{
+			SourceIP: strings.TrimSpace(x.Row.SourceIP), Count: x.Row.Count,
+			Disposition: lower(x.Row.Eval.Disposition), DKIM: lower(x.Row.Eval.DKIM), SPF: lower(x.Row.Eval.SPF),
+			HeaderFrom: lower(x.Identifiers.HeaderFrom), EnvelopeFrom: lower(x.Identifiers.EnvelopeFrom),
+		}
+		for _, d := range x.Auth.DKIM {
+			rec.DKIMAuth = append(rec.DKIMAuth, AuthResult{Domain: lower(d.Domain), Selector: d.Selector, Result: lower(d.Result)})
+		}
+		for _, s := range x.Auth.SPF {
+			rec.SPFAuth = append(rec.SPFAuth, AuthResult{Domain: lower(s.Domain), Result: lower(s.Result)})
+		}
+		r.Records = append(r.Records, rec)
+	}
+	return r, nil
+}
+
+func lower(s string) string { return strings.ToLower(strings.TrimSpace(s)) }
+
+// unwrap peels gzip, zip and email layers until XML is left.
+func unwrap(data []byte, depth int) ([]byte, error) {
+	if depth > 4 {
+		return nil, errors.New("report is wrapped too deeply")
+	}
+	switch {
+	case bytes.HasPrefix(data, []byte{0x1f, 0x8b}):
+		zr, err := gzip.NewReader(bytes.NewReader(data))
+		if err != nil {
+			return nil, err
+		}
+		defer zr.Close()
+		return readLimited(zr, depth)
+	case bytes.HasPrefix(data, []byte("PK\x03\x04")):
+		zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+		if err != nil {
+			return nil, err
+		}
+		for _, f := range zr.File {
+			if f.FileInfo().IsDir() {
+				continue
+			}
+			rc, err := f.Open()
+			if err != nil {
+				return nil, err
+			}
+			out, err := readLimited(rc, depth)
+			rc.Close()
+			return out, err
+		}
+		return nil, errors.New("empty zip")
+	}
+	trimmed := bytes.TrimSpace(data)
+	if bytes.HasPrefix(trimmed, []byte("<")) {
+		return trimmed, nil
+	}
+	return fromEmail(data, depth)
+}
+
+func readLimited(r io.Reader, depth int) ([]byte, error) {
+	b, err := io.ReadAll(io.LimitReader(r, maxXML+1))
+	if err != nil {
+		return nil, err
+	}
+	if len(b) > maxXML {
+		return nil, errors.New("report is too large")
+	}
+	return unwrap(b, depth+1)
+}
+
+// fromEmail finds the report attachment in a message.
+func fromEmail(data []byte, depth int) ([]byte, error) {
+	m, err := mail.ReadMessage(bytes.NewReader(data))
+	if err != nil {
+		return nil, errors.New("not a DMARC report: unrecognised format")
+	}
+	b, err := fromPart(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body, depth)
+	if err != nil {
+		return nil, err
+	}
+	if b == nil {
+		return nil, errors.New("no report attached to the email")
+	}
+	return b, nil
+}
+
+func fromPart(contentType, encoding string, body io.Reader, depth int) ([]byte, error) {
+	mt, params, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		mt = "text/plain"
+	}
+	if strings.HasPrefix(mt, "multipart/") {
+		mr := multipart.NewReader(body, params["boundary"])
+		for {
+			p, err := mr.NextPart()
+			if err != nil {
+				return nil, nil
+			}
+			b, err := fromPart(p.Header.Get("Content-Type"), p.Header.Get("Content-Transfer-Encoding"), p, depth)
+			if err != nil || b != nil {
+				return b, err
+			}
+		}
+	}
+	switch mt {
+	case "application/gzip", "application/x-gzip", "application/zip", "application/x-zip-compressed", "application/xml", "text/xml", "application/octet-stream":
+		raw, err := io.ReadAll(io.LimitReader(decodeBody(encoding, body), maxXML+1))
+		if err != nil {
+			return nil, err
+		}
+		return unwrap(raw, depth+1)
+	}
+	return nil, nil
+}
+
+func decodeBody(encoding string, r io.Reader) io.Reader {
+	switch strings.ToLower(strings.TrimSpace(encoding)) {
+	case "base64":
+		// mail wraps base64 in lines, which the decoder ignores when it is given the line breaks stripped
+		b, _ := io.ReadAll(io.LimitReader(r, maxXML*2))
+		clean := strings.Map(func(c rune) rune {
+			if c == '\r' || c == '\n' || c == ' ' || c == '\t' {
+				return -1
+			}
+			return c
+		}, string(b))
+		out, _ := base64.StdEncoding.DecodeString(clean)
+		return bytes.NewReader(out)
+	case "quoted-printable":
+		return quotedprintable.NewReader(r)
+	}
+	return r
+}

dmarc/dmarc_test.go

@@ -0,0 +1,157 @@
+package dmarc
+
+import (
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"strings"
+	"testing"
+)
+
+const sample = `<?xml version="1.0" encoding="UTF-8" ?>
+<feedback>
+  <report_metadata>
+    <org_name>google.com</org_name>
+    <email>noreply-dmarc-support@google.com</email>
+    <report_id>12345</report_id>
+    <date_range><begin>1700000000</begin><end>1700086399</end></date_range>
+  </report_metadata>
+  <policy_published>
+    <domain>Example.COM</domain>
+    <adkim>r</adkim><aspf>r</aspf><p>none</p><sp>none</sp><pct>100</pct>
+  </policy_published>
+  <record>
+    <row>
+      <source_ip>203.0.113.7</source_ip><count>12</count>
+      <policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>fail</spf></policy_evaluated>
+    </row>
+    <identifiers><header_from>example.com</header_from><envelope_from>bounce.example.com</envelope_from></identifiers>
+    <auth_results>
+      <dkim><domain>example.com</domain><selector>s1</selector><result>pass</result></dkim>
+      <spf><domain>bounce.example.com</domain><result>pass</result></spf>
+    </auth_results>
+  </record>
+  <record>
+    <row>
+      <source_ip>198.51.100.9</source_ip><count>3</count>
+      <policy_evaluated><disposition>quarantine</disposition><dkim>fail</dkim><spf>fail</spf></policy_evaluated>
+    </row>
+    <identifiers><header_from>example.com</header_from></identifiers>
+    <auth_results>
+      <spf><domain>example.com</domain><result>softfail</result></spf>
+    </auth_results>
+  </record>
+</feedback>`
+
+func check(t *testing.T, r *Report) {
+	t.Helper()
+	if r.Org != "google.com" || r.ReportID != "12345" || r.Domain != "example.com" {
+		t.Fatalf("metadata: %+v", r)
+	}
+	if r.Begin.Unix() != 1700000000 || r.End.Unix() != 1700086399 {
+		t.Fatalf("range: %v %v", r.Begin, r.End)
+	}
+	if r.Policy.P != "none" || r.Policy.Pct != 100 || r.Policy.ADKIM != "r" {
+		t.Fatalf("policy: %+v", r.Policy)
+	}
+	if len(r.Records) != 2 {
+		t.Fatalf("records: %d", len(r.Records))
+	}
+	a, b := r.Records[0], r.Records[1]
+	if a.SourceIP != "203.0.113.7" || a.Count != 12 || !a.Passed() || a.EnvelopeFrom != "bounce.example.com" {
+		t.Fatalf("first record: %+v", a)
+	}
+	if len(a.DKIMAuth) != 1 || a.DKIMAuth[0].Selector != "s1" || len(a.SPFAuth) != 1 {
+		t.Fatalf("auth results: %+v", a)
+	}
+	if b.Passed() || b.Disposition != "quarantine" || b.SPFAuth[0].Result != "softfail" {
+		t.Fatalf("second record: %+v", b)
+	}
+}
+
+func gz(t *testing.T, s string) []byte {
+	var buf bytes.Buffer
+	w := gzip.NewWriter(&buf)
+	w.Write([]byte(s))
+	w.Close()
+	return buf.Bytes()
+}
+
+func zipped(t *testing.T, s string) []byte {
+	var buf bytes.Buffer
+	w := zip.NewWriter(&buf)
+	f, _ := w.Create("report.xml")
+	f.Write([]byte(s))
+	w.Close()
+	return buf.Bytes()
+}
+
+func email(contentType, encoding string, body []byte) []byte {
+	b64 := base64.StdEncoding.EncodeToString(body)
+	var wrapped strings.Builder
+	for len(b64) > 76 {
+		wrapped.WriteString(b64[:76] + "\r\n")
+		b64 = b64[76:]
+	}
+	wrapped.WriteString(b64 + "\r\n")
+	return []byte("From: noreply@google.com\r\nSubject: Report Domain: example.com\r\nMIME-Version: 1.0\r\n" +
+		"Content-Type: multipart/mixed; boundary=\"XX\"\r\n\r\n--XX\r\nContent-Type: text/plain\r\n\r\nSee attached.\r\n" +
+		"--XX\r\nContent-Type: " + contentType + "\r\nContent-Transfer-Encoding: " + encoding + "\r\n\r\n" + wrapped.String() + "--XX--\r\n")
+}
+
+func TestParseForms(t *testing.T) {
+	forms := map[string][]byte{
+		"xml":        []byte("\n" + sample),
+		"gzip":       gz(t, sample),
+		"zip":        zipped(t, sample),
+		"mail+gzip":  email("application/gzip", "base64", gz(t, sample)),
+		"mail+zip":   email("application/zip; name=r.zip", "base64", zipped(t, sample)),
+		"mail+xml":   email("text/xml", "base64", []byte(sample)),
+		"mail+octet": email("application/octet-stream", "base64", gz(t, sample)),
+	}
+	for name, data := range forms {
+		t.Run(name, func(t *testing.T) {
+			r, err := Parse(data)
+			if err != nil {
+				t.Fatal(err)
+			}
+			check(t, r)
+		})
+	}
+}
+
+func TestParseRejects(t *testing.T) {
+	for name, data := range map[string][]byte{
+		"empty":       nil,
+		"text":        []byte("hello, this is not a report"),
+		"other xml":   []byte("<html><body/></html>"),
+		"no id":       []byte(strings.Replace(sample, "<report_id>12345</report_id>", "", 1)),
+		"mail no att": []byte("Subject: x\r\n\r\njust text\r\n"),
+		"bad gzip":    {0x1f, 0x8b, 0, 0},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if r, err := Parse(data); err == nil {
+				t.Fatalf("expected an error, got %+v", r)
+			}
+		})
+	}
+}
+
+func TestParseBombIsRefused(t *testing.T) {
+	big := "<feedback>" + strings.Repeat("x", maxXML+10) + "</feedback>"
+	if _, err := Parse(gz(t, big)); err == nil || !strings.Contains(err.Error(), "too large") {
+		t.Fatalf("want too large, got %v", err)
+	}
+}
+
+func TestPctDefaultsAndClamps(t *testing.T) {
+	r, err := Parse([]byte(strings.Replace(sample, "<pct>100</pct>", "<pct>25</pct>", 1)))
+	if err != nil || r.Policy.Pct != 25 {
+		t.Fatalf("pct 25: %v %+v", err, r)
+	}
+	r, err = Parse([]byte(strings.Replace(sample, "<pct>100</pct>", "", 1)))
+	if err != nil || r.Policy.Pct != 100 {
+		t.Fatalf("missing pct: %v %+v", err, r)
+	}
+}

model/model.go

@@ -241,6 +241,58 @@ type MailCheck struct {
 	HeloResolves   bool              `json:"heloResolves"`   // the HELO name resolves back to the source IP
 	Port25         string            `json:"port25"`         // "open" or why not
 	DKIM           map[string]string `json:"dkim"`           // key ID -> "published", "missing" or "mismatch"
+
+	Domains    map[string]DomainCheck `json:"domains,omitempty"`    // sending domain -> its DNS checks
+	Blocklists []string               `json:"blocklists,omitempty"` // DNS blocklists that list the source IP
+	Listable   int                    `json:"listable,omitempty"`   // how many blocklists answered, so "none listed" means something
+	Outcomes   *MailOutcomes          `json:"outcomes,omitempty"`   // what happened to the mail this egress sent
+}
+
+// Finding statuses.
+const (
+	FindingPass = "pass"
+	FindingWarn = "warn"
+	FindingFail = "fail"
+	FindingNone = "none" // nothing published
+)
+
+// MailFinding is the result of one deliverability check, with what to do about it.
+type MailFinding struct {
+	Status string `json:"status"`
+	Detail string `json:"detail"`
+	Hint   string `json:"hint,omitempty"`
+}
+
+// DomainCheck is how the DNS of one sending domain looks to a receiving server.
+type DomainCheck struct {
+	SPF    MailFinding `json:"spf"`
+	DMARC  MailFinding `json:"dmarc"`
+	MTASTS MailFinding `json:"mtaSts"`
+}
+
+// MailOutcomes counts what became of the mail an egress handled over a window.
+type MailOutcomes struct {
+	Since     time.Time               `json:"since"`
+	Delivered int                     `json:"delivered"`
+	Deferred  int                     `json:"deferred"`           // 4xx: the receiver asked us to try later
+	Failed    int                     `json:"failed"`             // 5xx: refused for good
+	ByDomain  map[string]OutcomeCount `json:"byDomain,omitempty"` // recipient domain
+	Recent    []OutcomeFailure        `json:"recent,omitempty"`   // latest problems, newest first
+}
+
+// OutcomeCount is delivered/deferred/failed for one recipient domain.
+type OutcomeCount struct {
+	Delivered int `json:"delivered"`
+	Deferred  int `json:"deferred"`
+	Failed    int `json:"failed"`
+}
+
+// OutcomeFailure is one deferral or refusal, for the dashboard to explain.
+type OutcomeFailure struct {
+	Time   time.Time `json:"time"`
+	Domain string    `json:"domain"`
+	Code   int       `json:"code"`
+	Text   string    `json:"text"`
 }
 
 // ---------- edge <-> hammerhead ----------