Check SPF, DMARC, MTA-STS and DNS blocklists per sending domain; count delivery outcomes
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
VeillaxandClaude Sonnet 5.5 committed
1 parent
b46e0813fe
commit
3a86254cb6
6 files changed
+864
No files matched your search
@@ -0,0 +1,473 @@
|
||||
package blacktip
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.veillax.com/Veillax/reef/model"
|
||||
)
|
||||
|
||||
// Deliverability checks for each sending domain and source IP: does SPF
|
||||
// authorize the IP, is there a DMARC policy worth enforcing, does the domain
|
||||
// publish MTA-STS, and is the IP on a DNS blocklist. Every finding carries a
|
||||
// hint saying what to change, since the people reading it run a mail server,
|
||||
// not a DNS course.
|
||||
|
||||
// defaultBlocklists are free-for-small-volume DNSBLs whose "listed" answers are reliable.
|
||||
var defaultBlocklists = []string{"zen.spamhaus.org", "bl.spamcop.net", "b.barracudacentral.org", "psbl.surriel.com"}
|
||||
|
||||
func (b *Blacktip) lookupIPs(name string) ([]net.IP, error) {
|
||||
if b.cfg.MailLookupIP != nil {
|
||||
return b.cfg.MailLookupIP(name)
|
||||
}
|
||||
addrs, err := net.LookupIP(name)
|
||||
return addrs, err
|
||||
}
|
||||
|
||||
func (b *Blacktip) lookupMXHosts(domain string) ([]string, error) {
|
||||
if b.cfg.MailLookupMX != nil {
|
||||
return b.cfg.MailLookupMX(domain)
|
||||
}
|
||||
mxs, err := net.LookupMX(domain)
|
||||
var out []string
|
||||
for _, mx := range mxs {
|
||||
out = append(out, strings.TrimSuffix(mx.Host, "."))
|
||||
}
|
||||
if len(out) > 0 {
|
||||
err = nil // a partial answer is still an answer
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
func dnsNotFound(err error) bool {
|
||||
var de *net.DNSError
|
||||
return errors.As(err, &de) && de.IsNotFound
|
||||
}
|
||||
|
||||
// ---------- SPF (RFC 7208) ----------
|
||||
|
||||
const spfLookupLimit = 10
|
||||
|
||||
type spfEval struct {
|
||||
b *Blacktip
|
||||
ip net.IP
|
||||
lookups int
|
||||
}
|
||||
|
||||
var errSPFPerm = errors.New("permerror")
|
||||
|
||||
// spfRecord returns the domain's SPF record text, or "" if it has none.
|
||||
func (b *Blacktip) spfRecord(domain string) (string, error) {
|
||||
txts, err := b.lookupTXT(domain)
|
||||
if err != nil && !dnsNotFound(err) && len(txts) == 0 {
|
||||
return "", err
|
||||
}
|
||||
rec := ""
|
||||
for _, t := range txts {
|
||||
low := strings.ToLower(strings.TrimSpace(t))
|
||||
if low == "v=spf1" || strings.HasPrefix(low, "v=spf1 ") {
|
||||
if rec != "" {
|
||||
return "", fmt.Errorf("%w: %s publishes more than one SPF record", errSPFPerm, domain)
|
||||
}
|
||||
rec = strings.TrimSpace(t)
|
||||
}
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// check evaluates domain's record for e.ip and returns the qualifier that
|
||||
// matched ("pass", "fail", "softfail", "neutral") or "none" with no record.
|
||||
func (e *spfEval) check(domain string, depth int) (string, error) {
|
||||
if depth > spfLookupLimit {
|
||||
return "", fmt.Errorf("%w: SPF includes nest too deeply", errSPFPerm)
|
||||
}
|
||||
rec, err := e.b.spfRecord(domain)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if rec == "" {
|
||||
return "none", nil
|
||||
}
|
||||
redirect := ""
|
||||
for _, term := range strings.Fields(rec)[1:] {
|
||||
low := strings.ToLower(term)
|
||||
if strings.HasPrefix(low, "redirect=") {
|
||||
redirect = term[len("redirect="):]
|
||||
continue
|
||||
}
|
||||
if strings.Contains(low, "=") && !strings.ContainsAny(low[:strings.IndexByte(low, '=')], ":/") {
|
||||
continue // an unknown modifier such as exp=
|
||||
}
|
||||
q, mech := "pass", low
|
||||
switch low[0] {
|
||||
case '+', '-', '~', '?':
|
||||
q = map[byte]string{'+': "pass", '-': "fail", '~': "softfail", '?': "neutral"}[low[0]]
|
||||
mech = low[1:]
|
||||
}
|
||||
base, cidr4, cidr6 := mech, -1, -1
|
||||
if !strings.HasPrefix(mech, "ip4:") && !strings.HasPrefix(mech, "ip6:") {
|
||||
if i := strings.IndexByte(mech, '/'); i >= 0 { // "a/24", "mx:host/24//64"
|
||||
var ok bool
|
||||
if cidr4, cidr6, ok = parsePrefixes(mech[i+1:]); !ok {
|
||||
return "", fmt.Errorf("%w: bad prefix in %q", errSPFPerm, term)
|
||||
}
|
||||
base = mech[:i]
|
||||
}
|
||||
}
|
||||
name, arg, _ := strings.Cut(base, ":")
|
||||
matched := false
|
||||
switch name {
|
||||
case "all":
|
||||
matched = true
|
||||
case "ip4", "ip6":
|
||||
matched = ipInSpec(e.ip, arg)
|
||||
case "a", "mx", "include", "exists", "ptr":
|
||||
if name == "ptr" {
|
||||
continue // discouraged by the RFC and slow: treated as not matching
|
||||
}
|
||||
if e.lookups++; e.lookups > spfLookupLimit {
|
||||
return "", fmt.Errorf("%w: more than %d DNS lookups", errSPFPerm, spfLookupLimit)
|
||||
}
|
||||
target := domain
|
||||
if arg != "" {
|
||||
target = arg
|
||||
}
|
||||
switch name {
|
||||
case "include":
|
||||
if arg == "" {
|
||||
return "", fmt.Errorf("%w: include without a domain", errSPFPerm)
|
||||
}
|
||||
res, err := e.check(target, depth+1)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if res == "none" {
|
||||
return "", fmt.Errorf("%w: included domain %s has no SPF record", errSPFPerm, target)
|
||||
}
|
||||
matched = res == "pass"
|
||||
case "a":
|
||||
matched = e.hostMatches(target, cidr4, cidr6)
|
||||
case "exists":
|
||||
ips, _ := e.b.lookupIPs(target)
|
||||
matched = len(ips) > 0
|
||||
case "mx":
|
||||
hosts, _ := e.b.lookupMXHosts(target)
|
||||
for _, h := range hosts {
|
||||
if e.lookups++; e.lookups > spfLookupLimit {
|
||||
return "", fmt.Errorf("%w: more than %d DNS lookups", errSPFPerm, spfLookupLimit)
|
||||
}
|
||||
if e.hostMatches(h, cidr4, cidr6) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if matched {
|
||||
return q, nil
|
||||
}
|
||||
}
|
||||
if redirect != "" {
|
||||
if e.lookups++; e.lookups > spfLookupLimit {
|
||||
return "", fmt.Errorf("%w: more than %d DNS lookups", errSPFPerm, spfLookupLimit)
|
||||
}
|
||||
res, err := e.check(redirect, depth+1)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if res == "none" {
|
||||
return "", fmt.Errorf("%w: redirect target %s has no SPF record", errSPFPerm, redirect)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
return "neutral", nil
|
||||
}
|
||||
|
||||
// parsePrefixes reads "24", "24//64" or "/64" (IPv4 and IPv6 prefix lengths, -1 when absent).
|
||||
func parsePrefixes(s string) (v4, v6 int, ok bool) {
|
||||
v4, v6 = -1, -1
|
||||
if strings.HasPrefix(s, "/") && !strings.HasPrefix(s, "//") {
|
||||
s = "/" + s // "a//64" arrives here as "/64"
|
||||
}
|
||||
a, b, has6 := strings.Cut(s, "//")
|
||||
if a != "" {
|
||||
n, err := strconv.Atoi(a)
|
||||
if err != nil || n < 0 || n > 32 {
|
||||
return 0, 0, false
|
||||
}
|
||||
v4 = n
|
||||
}
|
||||
if has6 {
|
||||
n, err := strconv.Atoi(b)
|
||||
if err != nil || n < 0 || n > 128 {
|
||||
return 0, 0, false
|
||||
}
|
||||
v6 = n
|
||||
}
|
||||
return v4, v6, true
|
||||
}
|
||||
|
||||
func (e *spfEval) hostMatches(host string, cidr4, cidr6 int) bool {
|
||||
ips, _ := e.b.lookupIPs(host)
|
||||
for _, ip := range ips {
|
||||
bits := cidr4
|
||||
if ip.To4() == nil {
|
||||
bits = cidr6
|
||||
}
|
||||
if bits < 0 {
|
||||
if ip.Equal(e.ip) {
|
||||
return true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, n, err := net.ParseCIDR(ip.String() + "/" + strconv.Itoa(bits)); err == nil && n.Contains(e.ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ipInSpec(ip net.IP, spec string) bool {
|
||||
if strings.Contains(spec, "/") {
|
||||
_, n, err := net.ParseCIDR(spec)
|
||||
return err == nil && n.Contains(ip)
|
||||
}
|
||||
p := net.ParseIP(spec)
|
||||
return p != nil && p.Equal(ip)
|
||||
}
|
||||
|
||||
// checkSPF says whether the domain's SPF record authorizes the source IP.
|
||||
func (b *Blacktip) checkSPF(domain, sourceIP string) model.MailFinding {
|
||||
ip := net.ParseIP(sourceIP)
|
||||
if ip == nil {
|
||||
return model.MailFinding{Status: model.FindingWarn, Detail: "the source address isn't an IP address"}
|
||||
}
|
||||
hint := fmt.Sprintf("Publish a TXT record on %s: \"v=spf1 ip4:%s ~all\". If other services also send as %s, add their include: entries to the same record.", domain, sourceIP, domain)
|
||||
if ip.To4() == nil {
|
||||
hint = strings.Replace(hint, "ip4:", "ip6:", 1)
|
||||
}
|
||||
e := &spfEval{b: b, ip: ip}
|
||||
res, err := e.check(domain, 0)
|
||||
switch {
|
||||
case errors.Is(err, errSPFPerm):
|
||||
return model.MailFinding{Status: model.FindingFail, Detail: "SPF record is invalid: " + strings.TrimPrefix(err.Error(), "permerror: "),
|
||||
Hint: "Receivers treat an invalid SPF record as a failure. Fix the record on " + domain + "; SPF allows at most 10 DNS lookups across include:, a, mx and redirect, so flatten some includes into ip4: entries."}
|
||||
case err != nil:
|
||||
return model.MailFinding{Status: model.FindingWarn, Detail: "couldn't read the SPF record: " + err.Error()}
|
||||
case res == "none":
|
||||
return model.MailFinding{Status: model.FindingNone, Detail: "no SPF record published on " + domain, Hint: hint}
|
||||
case res == "pass":
|
||||
rec, _ := b.spfRecord(domain)
|
||||
if low := strings.ToLower(rec); strings.HasSuffix(low, " +all") || strings.HasSuffix(low, " ?all") || strings.HasSuffix(low, " all") {
|
||||
return model.MailFinding{Status: model.FindingWarn, Detail: "SPF authorizes " + sourceIP + " but ends in a permissive all, which lets anyone send as " + domain,
|
||||
Hint: "End the record with ~all (or -all once you're sure every sender is listed)."}
|
||||
}
|
||||
return model.MailFinding{Status: model.FindingPass, Detail: "SPF authorizes " + sourceIP}
|
||||
default:
|
||||
return model.MailFinding{Status: model.FindingFail, Detail: fmt.Sprintf("SPF for %s does not authorize %s (result: %s)", domain, sourceIP, res),
|
||||
Hint: fmt.Sprintf("Add ip4:%s to the v=spf1 record on %s, before the final all.", sourceIP, domain)}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- DMARC (RFC 7489) ----------
|
||||
|
||||
// dmarcRecord finds the DMARC record that applies to domain: its own, else its
|
||||
// parent's (a cheap stand-in for the organizational domain, which would need
|
||||
// the public suffix list).
|
||||
func (b *Blacktip) dmarcRecord(domain string) (rec, at string, err error) {
|
||||
for d := domain; strings.Count(d, ".") >= 1; {
|
||||
txts, lerr := b.lookupTXT("_dmarc." + d)
|
||||
if lerr != nil && !dnsNotFound(lerr) && len(txts) == 0 {
|
||||
return "", "", lerr
|
||||
}
|
||||
for _, t := range txts {
|
||||
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(t)), "v=dmarc1") {
|
||||
return strings.TrimSpace(t), d, nil
|
||||
}
|
||||
}
|
||||
_, d, _ = strings.Cut(d, ".")
|
||||
}
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
func dmarcTags(rec string) map[string]string {
|
||||
tags := map[string]string{}
|
||||
for _, f := range strings.Split(rec, ";") {
|
||||
if k, v, ok := strings.Cut(strings.TrimSpace(f), "="); ok {
|
||||
tags[strings.ToLower(strings.TrimSpace(k))] = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
func (b *Blacktip) checkDMARC(domain string) model.MailFinding {
|
||||
rec, at, err := b.dmarcRecord(domain)
|
||||
if err != nil {
|
||||
return model.MailFinding{Status: model.FindingWarn, Detail: "couldn't read the DMARC record: " + err.Error()}
|
||||
}
|
||||
if rec == "" {
|
||||
return model.MailFinding{Status: model.FindingNone, Detail: "no DMARC record published for " + domain,
|
||||
Hint: fmt.Sprintf("Publish a TXT record on _dmarc.%s: \"v=DMARC1; p=none; rua=mailto:dmarc@%s\". Start with p=none, read the reports, then tighten it. Gmail and Yahoo now require a DMARC record from bulk senders.", domain, domain)}
|
||||
}
|
||||
t := dmarcTags(rec)
|
||||
where := ""
|
||||
if at != domain {
|
||||
where = " (inherited from " + at + ")"
|
||||
}
|
||||
p := strings.ToLower(t["p"])
|
||||
if sp := strings.ToLower(t["sp"]); at != domain && sp != "" {
|
||||
p = sp
|
||||
}
|
||||
switch {
|
||||
case p != "none" && p != "quarantine" && p != "reject":
|
||||
return model.MailFinding{Status: model.FindingFail, Detail: "DMARC record has no valid p= policy" + where,
|
||||
Hint: "Set p=none, p=quarantine or p=reject in the _dmarc." + at + " record; receivers ignore a record without one."}
|
||||
case t["rua"] == "":
|
||||
return model.MailFinding{Status: model.FindingWarn, Detail: "DMARC policy is " + p + " but there is no rua= address, so you'd get no reports" + where,
|
||||
Hint: "Add rua=mailto:dmarc@" + domain + " to the record, then upload the reports you receive on the Mail page."}
|
||||
case p == "none":
|
||||
return model.MailFinding{Status: model.FindingWarn, Detail: "DMARC is in monitoring mode (p=none)" + where,
|
||||
Hint: "Once the reports show only your own legitimate mail passing, move to p=quarantine and then p=reject."}
|
||||
}
|
||||
return model.MailFinding{Status: model.FindingPass, Detail: "DMARC policy is " + p + where}
|
||||
}
|
||||
|
||||
// ---------- MTA-STS for the sending domain ----------
|
||||
|
||||
func (b *Blacktip) checkMTASTS(domain string) model.MailFinding {
|
||||
pol := b.stsFor(domain)
|
||||
if pol == nil {
|
||||
return model.MailFinding{Status: model.FindingNone, Detail: "no MTA-STS policy for " + domain,
|
||||
Hint: "Optional but recommended: publish a TXT record on _mta-sts." + domain + " (\"v=STSv1; id=1\") and serve a policy at https://mta-sts." + domain + "/.well-known/mta-sts.txt so senders must use TLS to reach you."}
|
||||
}
|
||||
switch pol.Mode {
|
||||
case "enforce":
|
||||
return model.MailFinding{Status: model.FindingPass, Detail: "MTA-STS is enforced"}
|
||||
case "testing":
|
||||
return model.MailFinding{Status: model.FindingWarn, Detail: "MTA-STS is in testing mode",
|
||||
Hint: "Once TLS reports look clean, change mode: testing to mode: enforce in the policy file and bump the id in the DNS record."}
|
||||
}
|
||||
return model.MailFinding{Status: model.FindingNone, Detail: "MTA-STS policy is switched off (mode: none)"}
|
||||
}
|
||||
|
||||
// ---------- blocklists ----------
|
||||
|
||||
func (b *Blacktip) dnsblQuery(name string) ([]string, error) {
|
||||
if b.cfg.MailLookupDNSBL != nil {
|
||||
return b.cfg.MailLookupDNSBL(name)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
|
||||
defer cancel()
|
||||
addrs, err := net.DefaultResolver.LookupHost(ctx, name)
|
||||
if dnsNotFound(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return addrs, err
|
||||
}
|
||||
|
||||
// reverseIPv4 turns 192.0.2.1 into 1.2.0.192, the form DNSBLs are queried in.
|
||||
func reverseIPv4(ip net.IP) string {
|
||||
v4 := ip.To4()
|
||||
if v4 == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%d.%d.%d.%d", v4[3], v4[2], v4[1], v4[0])
|
||||
}
|
||||
|
||||
// checkBlocklists asks each DNSBL about the IP. listed are zones that list it;
|
||||
// answered counts the zones that gave a usable answer (listing refusals for
|
||||
// open resolvers, answered as 127.255.255.x, don't count as either).
|
||||
func (b *Blacktip) checkBlocklists(sourceIP string) (listed []string, answered int) {
|
||||
rev := reverseIPv4(net.ParseIP(sourceIP))
|
||||
if rev == "" {
|
||||
return nil, 0 // the common blocklists only cover IPv4
|
||||
}
|
||||
zones := b.cfg.MailBlocklists
|
||||
if zones == nil {
|
||||
zones = defaultBlocklists
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for _, z := range zones {
|
||||
wg.Add(1)
|
||||
go func(z string) {
|
||||
defer wg.Done()
|
||||
addrs, err := b.dnsblQuery(rev + "." + z)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
hit, bad := false, false
|
||||
for _, a := range addrs {
|
||||
if strings.HasPrefix(a, "127.255.255.") {
|
||||
bad = true // "query refused": the zone won't answer from this resolver
|
||||
} else if strings.HasPrefix(a, "127.") {
|
||||
hit = true
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if hit {
|
||||
listed = append(listed, z)
|
||||
}
|
||||
if !bad || hit {
|
||||
answered++
|
||||
}
|
||||
}(z)
|
||||
}
|
||||
wg.Wait()
|
||||
sort.Strings(listed)
|
||||
return listed, answered
|
||||
}
|
||||
|
||||
// ---------- putting it together ----------
|
||||
|
||||
// sendingDomains are the domains the policy has DKIM keys for: the ones it signs mail as.
|
||||
func sendingDomains(pol model.MailPolicy) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, k := range pol.DKIM {
|
||||
d := strings.ToLower(k.Domain)
|
||||
if d != "" && !seen[d] {
|
||||
seen[d] = true
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// checkDomains runs the per-domain DNS checks for a policy's sending domains.
|
||||
func (b *Blacktip) checkDomains(pol model.MailPolicy) map[string]model.DomainCheck {
|
||||
doms := sendingDomains(pol)
|
||||
if len(doms) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := map[string]model.DomainCheck{}
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for _, d := range doms {
|
||||
wg.Add(1)
|
||||
go func(d string) {
|
||||
defer wg.Done()
|
||||
c := model.DomainCheck{DMARC: b.checkDMARC(d), MTASTS: b.checkMTASTS(d)}
|
||||
if pol.Smarthost != "" {
|
||||
c.SPF = model.MailFinding{Status: model.FindingWarn, Detail: "not checked: mail leaves through the smarthost " + pol.Smarthost,
|
||||
Hint: "The receiving server sees the smarthost's address, so " + d + "'s SPF record must authorize that provider instead."}
|
||||
} else {
|
||||
c.SPF = b.checkSPF(d, pol.SourceIP)
|
||||
}
|
||||
mu.Lock()
|
||||
out[d] = c
|
||||
mu.Unlock()
|
||||
}(d)
|
||||
}
|
||||
wg.Wait()
|
||||
return out
|
||||
}
|
||||
Reference in new issue
Block a user