diff --git a/dmarc/dmarc.go b/dmarc/dmarc.go new file mode 100644 index 0000000..9100c5a --- /dev/null +++ b/dmarc/dmarc.go @@ -0,0 +1,274 @@ +// Package dmarc parses DMARC aggregate (rua) reports (RFC 7489 appendix A). +// Receivers send them as XML, usually gzip or zip compressed, usually as an +// email attachment; Parse accepts any of those forms. +package dmarc + +import ( + "archive/zip" + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/xml" + "errors" + "io" + "mime" + "mime/multipart" + "mime/quotedprintable" + "net/mail" + "strconv" + "strings" + "time" +) + +// maxXML caps a decompressed report; aggregate reports are small, so a larger one is a zip bomb. +const maxXML = 32 << 20 + +// Report is one receiver's summary of the mail it saw claiming to be from Domain. +type Report struct { + Org string `json:"org"` // the receiver that sent the report + Email string `json:"email"` + ReportID string `json:"reportId"` + Begin time.Time `json:"begin"` + End time.Time `json:"end"` + Domain string `json:"domain"` + Policy Policy `json:"policy"` + Records []Record `json:"records"` +} + +// Policy is the DMARC policy the receiver applied. +type Policy struct { + P string `json:"p"` // none, quarantine or reject + SP string `json:"sp"` // for subdomains + Pct int `json:"pct"` // percent of mail the policy applies to + ADKIM string `json:"adkim"` + ASPF string `json:"aspf"` +} + +// Record is one row: a sending IP and what happened to its mail. +type Record struct { + SourceIP string `json:"sourceIp"` + Count int `json:"count"` + Disposition string `json:"disposition"` // what the receiver did: none, quarantine, reject + DKIM string `json:"dkim"` // aligned DKIM result: pass or fail + SPF string `json:"spf"` // aligned SPF result: pass or fail + HeaderFrom string `json:"headerFrom"` + EnvelopeFrom string `json:"envelopeFrom,omitempty"` + DKIMAuth []AuthResult `json:"dkimAuth,omitempty"` + SPFAuth []AuthResult `json:"spfAuth,omitempty"` +} + +// AuthResult is a raw DKIM or SPF verdict, before alignment with the From domain. +type AuthResult struct { + Domain string `json:"domain"` + Selector string `json:"selector,omitempty"` + Result string `json:"result"` +} + +// Passed reports whether the row's mail passed DMARC: either aligned check passing is enough. +func (r Record) Passed() bool { return r.DKIM == "pass" || r.SPF == "pass" } + +type xmlFeedback struct { + Meta struct { + Org string `xml:"org_name"` + Email string `xml:"email"` + ReportID string `xml:"report_id"` + Range struct { + Begin int64 `xml:"begin"` + End int64 `xml:"end"` + } `xml:"date_range"` + } `xml:"report_metadata"` + Policy struct { + Domain string `xml:"domain"` + ADKIM string `xml:"adkim"` + ASPF string `xml:"aspf"` + P string `xml:"p"` + SP string `xml:"sp"` + Pct string `xml:"pct"` + } `xml:"policy_published"` + Records []struct { + Row struct { + SourceIP string `xml:"source_ip"` + Count int `xml:"count"` + Eval struct { + Disposition string `xml:"disposition"` + DKIM string `xml:"dkim"` + SPF string `xml:"spf"` + } `xml:"policy_evaluated"` + } `xml:"row"` + Identifiers struct { + HeaderFrom string `xml:"header_from"` + EnvelopeFrom string `xml:"envelope_from"` + } `xml:"identifiers"` + Auth struct { + DKIM []struct { + Domain string `xml:"domain"` + Selector string `xml:"selector"` + Result string `xml:"result"` + } `xml:"dkim"` + SPF []struct { + Domain string `xml:"domain"` + Result string `xml:"result"` + } `xml:"spf"` + } `xml:"auth_results"` + } `xml:"record"` +} + +// Parse reads a report from raw XML, gzip, zip, or a whole email carrying one of those. +func Parse(data []byte) (*Report, error) { + xmlData, err := unwrap(data, 0) + if err != nil { + return nil, err + } + var f xmlFeedback + if err := xml.Unmarshal(xmlData, &f); err != nil { + return nil, errors.New("not a DMARC report: " + err.Error()) + } + if f.Policy.Domain == "" { + return nil, errors.New("not a DMARC report: no published policy domain") + } + if strings.TrimSpace(f.Meta.ReportID) == "" { + return nil, errors.New("not a DMARC report: no report_id") + } + pct := 100 + if n, err := strconv.Atoi(strings.TrimSpace(f.Policy.Pct)); err == nil && n >= 0 && n <= 100 { + pct = n + } + r := &Report{ + Org: strings.TrimSpace(f.Meta.Org), Email: strings.TrimSpace(f.Meta.Email), ReportID: strings.TrimSpace(f.Meta.ReportID), + Begin: time.Unix(f.Meta.Range.Begin, 0).UTC(), + End: time.Unix(f.Meta.Range.End, 0).UTC(), + Domain: lower(f.Policy.Domain), + Policy: Policy{P: lower(f.Policy.P), SP: lower(f.Policy.SP), Pct: pct, ADKIM: lower(f.Policy.ADKIM), ASPF: lower(f.Policy.ASPF)}, + } + for _, x := range f.Records { + rec := Record{ + SourceIP: strings.TrimSpace(x.Row.SourceIP), Count: x.Row.Count, + Disposition: lower(x.Row.Eval.Disposition), DKIM: lower(x.Row.Eval.DKIM), SPF: lower(x.Row.Eval.SPF), + HeaderFrom: lower(x.Identifiers.HeaderFrom), EnvelopeFrom: lower(x.Identifiers.EnvelopeFrom), + } + for _, d := range x.Auth.DKIM { + rec.DKIMAuth = append(rec.DKIMAuth, AuthResult{Domain: lower(d.Domain), Selector: d.Selector, Result: lower(d.Result)}) + } + for _, s := range x.Auth.SPF { + rec.SPFAuth = append(rec.SPFAuth, AuthResult{Domain: lower(s.Domain), Result: lower(s.Result)}) + } + r.Records = append(r.Records, rec) + } + return r, nil +} + +func lower(s string) string { return strings.ToLower(strings.TrimSpace(s)) } + +// unwrap peels gzip, zip and email layers until XML is left. +func unwrap(data []byte, depth int) ([]byte, error) { + if depth > 4 { + return nil, errors.New("report is wrapped too deeply") + } + switch { + case bytes.HasPrefix(data, []byte{0x1f, 0x8b}): + zr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer zr.Close() + return readLimited(zr, depth) + case bytes.HasPrefix(data, []byte("PK\x03\x04")): + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, err + } + for _, f := range zr.File { + if f.FileInfo().IsDir() { + continue + } + rc, err := f.Open() + if err != nil { + return nil, err + } + out, err := readLimited(rc, depth) + rc.Close() + return out, err + } + return nil, errors.New("empty zip") + } + trimmed := bytes.TrimSpace(data) + if bytes.HasPrefix(trimmed, []byte("<")) { + return trimmed, nil + } + return fromEmail(data, depth) +} + +func readLimited(r io.Reader, depth int) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, maxXML+1)) + if err != nil { + return nil, err + } + if len(b) > maxXML { + return nil, errors.New("report is too large") + } + return unwrap(b, depth+1) +} + +// fromEmail finds the report attachment in a message. +func fromEmail(data []byte, depth int) ([]byte, error) { + m, err := mail.ReadMessage(bytes.NewReader(data)) + if err != nil { + return nil, errors.New("not a DMARC report: unrecognised format") + } + b, err := fromPart(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body, depth) + if err != nil { + return nil, err + } + if b == nil { + return nil, errors.New("no report attached to the email") + } + return b, nil +} + +func fromPart(contentType, encoding string, body io.Reader, depth int) ([]byte, error) { + mt, params, err := mime.ParseMediaType(contentType) + if err != nil { + mt = "text/plain" + } + if strings.HasPrefix(mt, "multipart/") { + mr := multipart.NewReader(body, params["boundary"]) + for { + p, err := mr.NextPart() + if err != nil { + return nil, nil + } + b, err := fromPart(p.Header.Get("Content-Type"), p.Header.Get("Content-Transfer-Encoding"), p, depth) + if err != nil || b != nil { + return b, err + } + } + } + switch mt { + case "application/gzip", "application/x-gzip", "application/zip", "application/x-zip-compressed", "application/xml", "text/xml", "application/octet-stream": + raw, err := io.ReadAll(io.LimitReader(decodeBody(encoding, body), maxXML+1)) + if err != nil { + return nil, err + } + return unwrap(raw, depth+1) + } + return nil, nil +} + +func decodeBody(encoding string, r io.Reader) io.Reader { + switch strings.ToLower(strings.TrimSpace(encoding)) { + case "base64": + // mail wraps base64 in lines, which the decoder ignores when it is given the line breaks stripped + b, _ := io.ReadAll(io.LimitReader(r, maxXML*2)) + clean := strings.Map(func(c rune) rune { + if c == '\r' || c == '\n' || c == ' ' || c == '\t' { + return -1 + } + return c + }, string(b)) + out, _ := base64.StdEncoding.DecodeString(clean) + return bytes.NewReader(out) + case "quoted-printable": + return quotedprintable.NewReader(r) + } + return r +} diff --git a/dmarc/dmarc_test.go b/dmarc/dmarc_test.go new file mode 100644 index 0000000..b1da0d4 --- /dev/null +++ b/dmarc/dmarc_test.go @@ -0,0 +1,157 @@ +package dmarc + +import ( + "archive/zip" + "bytes" + "compress/gzip" + "encoding/base64" + "strings" + "testing" +) + +const sample = ` + + + google.com + noreply-dmarc-support@google.com + 12345 + 17000000001700086399 + + + Example.COM + rr

none

none100 +
+ + + 203.0.113.712 + nonepassfail + + example.combounce.example.com + + example.coms1pass + bounce.example.compass + + + + + 198.51.100.93 + quarantinefailfail + + example.com + + example.comsoftfail + + +
` + +func check(t *testing.T, r *Report) { + t.Helper() + if r.Org != "google.com" || r.ReportID != "12345" || r.Domain != "example.com" { + t.Fatalf("metadata: %+v", r) + } + if r.Begin.Unix() != 1700000000 || r.End.Unix() != 1700086399 { + t.Fatalf("range: %v %v", r.Begin, r.End) + } + if r.Policy.P != "none" || r.Policy.Pct != 100 || r.Policy.ADKIM != "r" { + t.Fatalf("policy: %+v", r.Policy) + } + if len(r.Records) != 2 { + t.Fatalf("records: %d", len(r.Records)) + } + a, b := r.Records[0], r.Records[1] + if a.SourceIP != "203.0.113.7" || a.Count != 12 || !a.Passed() || a.EnvelopeFrom != "bounce.example.com" { + t.Fatalf("first record: %+v", a) + } + if len(a.DKIMAuth) != 1 || a.DKIMAuth[0].Selector != "s1" || len(a.SPFAuth) != 1 { + t.Fatalf("auth results: %+v", a) + } + if b.Passed() || b.Disposition != "quarantine" || b.SPFAuth[0].Result != "softfail" { + t.Fatalf("second record: %+v", b) + } +} + +func gz(t *testing.T, s string) []byte { + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + w.Write([]byte(s)) + w.Close() + return buf.Bytes() +} + +func zipped(t *testing.T, s string) []byte { + var buf bytes.Buffer + w := zip.NewWriter(&buf) + f, _ := w.Create("report.xml") + f.Write([]byte(s)) + w.Close() + return buf.Bytes() +} + +func email(contentType, encoding string, body []byte) []byte { + b64 := base64.StdEncoding.EncodeToString(body) + var wrapped strings.Builder + for len(b64) > 76 { + wrapped.WriteString(b64[:76] + "\r\n") + b64 = b64[76:] + } + wrapped.WriteString(b64 + "\r\n") + return []byte("From: noreply@google.com\r\nSubject: Report Domain: example.com\r\nMIME-Version: 1.0\r\n" + + "Content-Type: multipart/mixed; boundary=\"XX\"\r\n\r\n--XX\r\nContent-Type: text/plain\r\n\r\nSee attached.\r\n" + + "--XX\r\nContent-Type: " + contentType + "\r\nContent-Transfer-Encoding: " + encoding + "\r\n\r\n" + wrapped.String() + "--XX--\r\n") +} + +func TestParseForms(t *testing.T) { + forms := map[string][]byte{ + "xml": []byte("\n" + sample), + "gzip": gz(t, sample), + "zip": zipped(t, sample), + "mail+gzip": email("application/gzip", "base64", gz(t, sample)), + "mail+zip": email("application/zip; name=r.zip", "base64", zipped(t, sample)), + "mail+xml": email("text/xml", "base64", []byte(sample)), + "mail+octet": email("application/octet-stream", "base64", gz(t, sample)), + } + for name, data := range forms { + t.Run(name, func(t *testing.T) { + r, err := Parse(data) + if err != nil { + t.Fatal(err) + } + check(t, r) + }) + } +} + +func TestParseRejects(t *testing.T) { + for name, data := range map[string][]byte{ + "empty": nil, + "text": []byte("hello, this is not a report"), + "other xml": []byte(""), + "no id": []byte(strings.Replace(sample, "12345", "", 1)), + "mail no att": []byte("Subject: x\r\n\r\njust text\r\n"), + "bad gzip": {0x1f, 0x8b, 0, 0}, + } { + t.Run(name, func(t *testing.T) { + if r, err := Parse(data); err == nil { + t.Fatalf("expected an error, got %+v", r) + } + }) + } +} + +func TestParseBombIsRefused(t *testing.T) { + big := "" + strings.Repeat("x", maxXML+10) + "" + if _, err := Parse(gz(t, big)); err == nil || !strings.Contains(err.Error(), "too large") { + t.Fatalf("want too large, got %v", err) + } +} + +func TestPctDefaultsAndClamps(t *testing.T) { + r, err := Parse([]byte(strings.Replace(sample, "100", "25", 1))) + if err != nil || r.Policy.Pct != 25 { + t.Fatalf("pct 25: %v %+v", err, r) + } + r, err = Parse([]byte(strings.Replace(sample, "100", "", 1))) + if err != nil || r.Policy.Pct != 100 { + t.Fatalf("missing pct: %v %+v", err, r) + } +} diff --git a/model/model.go b/model/model.go index 162f279..c4a9833 100644 --- a/model/model.go +++ b/model/model.go @@ -242,6 +242,58 @@ type MailCheck struct { HeloResolves bool `json:"heloResolves"` // the HELO name resolves back to the source IP Port25 string `json:"port25"` // "open" or why not DKIM map[string]string `json:"dkim"` // key ID -> "published", "missing" or "mismatch" + + Domains map[string]DomainCheck `json:"domains,omitempty"` // sending domain -> its DNS checks + Blocklists []string `json:"blocklists,omitempty"` // DNS blocklists that list the source IP + Listable int `json:"listable,omitempty"` // how many blocklists answered, so "none listed" means something + Outcomes *MailOutcomes `json:"outcomes,omitempty"` // what happened to the mail this egress sent +} + +// Finding statuses. +const ( + FindingPass = "pass" + FindingWarn = "warn" + FindingFail = "fail" + FindingNone = "none" // nothing published +) + +// MailFinding is the result of one deliverability check, with what to do about it. +type MailFinding struct { + Status string `json:"status"` + Detail string `json:"detail"` + Hint string `json:"hint,omitempty"` +} + +// DomainCheck is how the DNS of one sending domain looks to a receiving server. +type DomainCheck struct { + SPF MailFinding `json:"spf"` + DMARC MailFinding `json:"dmarc"` + MTASTS MailFinding `json:"mtaSts"` +} + +// MailOutcomes counts what became of the mail an egress handled over a window. +type MailOutcomes struct { + Since time.Time `json:"since"` + Delivered int `json:"delivered"` + Deferred int `json:"deferred"` // 4xx: the receiver asked us to try later + Failed int `json:"failed"` // 5xx: refused for good + ByDomain map[string]OutcomeCount `json:"byDomain,omitempty"` // recipient domain + Recent []OutcomeFailure `json:"recent,omitempty"` // latest problems, newest first +} + +// OutcomeCount is delivered/deferred/failed for one recipient domain. +type OutcomeCount struct { + Delivered int `json:"delivered"` + Deferred int `json:"deferred"` + Failed int `json:"failed"` +} + +// OutcomeFailure is one deferral or refusal, for the dashboard to explain. +type OutcomeFailure struct { + Time time.Time `json:"time"` + Domain string `json:"domain"` + Code int `json:"code"` + Text string `json:"text"` } // ---------- edge <-> hammerhead ----------