Blacktip/deliverability.go

package blacktip

import (
	"context"
	"errors"
	"fmt"
	"net"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"git.veillax.com/Veillax/reef/model"
)

// Deliverability checks for each sending domain and source IP: does SPF
// authorize the IP, is there a DMARC policy worth enforcing, does the domain
// publish MTA-STS, and is the IP on a DNS blocklist. Every finding carries a
// hint saying what to change, since the people reading it run a mail server,
// not a DNS course.

// defaultBlocklists are free-for-small-volume DNSBLs whose "listed" answers are reliable.
var defaultBlocklists = []string{"zen.spamhaus.org", "bl.spamcop.net", "b.barracudacentral.org", "psbl.surriel.com"}

func (b *Blacktip) lookupIPs(name string) ([]net.IP, error) {
	if b.cfg.MailLookupIP != nil {
		return b.cfg.MailLookupIP(name)
	}
	addrs, err := net.LookupIP(name)
	return addrs, err
}

func (b *Blacktip) lookupMXHosts(domain string) ([]string, error) {
	if b.cfg.MailLookupMX != nil {
		return b.cfg.MailLookupMX(domain)
	}
	mxs, err := net.LookupMX(domain)
	var out []string
	for _, mx := range mxs {
		out = append(out, strings.TrimSuffix(mx.Host, "."))
	}
	if len(out) > 0 {
		err = nil // a partial answer is still an answer
	}
	return out, err
}

func dnsNotFound(err error) bool {
	var de *net.DNSError
	return errors.As(err, &de) && de.IsNotFound
}

// ---------- SPF (RFC 7208) ----------

const spfLookupLimit = 10

type spfEval struct {
	b       *Blacktip
	ip      net.IP
	lookups int
}

var errSPFPerm = errors.New("permerror")

// spfRecord returns the domain's SPF record text, or "" if it has none.
func (b *Blacktip) spfRecord(domain string) (string, error) {
	txts, err := b.lookupTXT(domain)
	if err != nil && !dnsNotFound(err) && len(txts) == 0 {
		return "", err
	}
	rec := ""
	for _, t := range txts {
		low := strings.ToLower(strings.TrimSpace(t))
		if low == "v=spf1" || strings.HasPrefix(low, "v=spf1 ") {
			if rec != "" {
				return "", fmt.Errorf("%w: %s publishes more than one SPF record", errSPFPerm, domain)
			}
			rec = strings.TrimSpace(t)
		}
	}
	return rec, nil
}

// check evaluates domain's record for e.ip and returns the qualifier that
// matched ("pass", "fail", "softfail", "neutral") or "none" with no record.
func (e *spfEval) check(domain string, depth int) (string, error) {
	if depth > spfLookupLimit {
		return "", fmt.Errorf("%w: SPF includes nest too deeply", errSPFPerm)
	}
	rec, err := e.b.spfRecord(domain)
	if err != nil {
		return "", err
	}
	if rec == "" {
		return "none", nil
	}
	redirect := ""
	for _, term := range strings.Fields(rec)[1:] {
		low := strings.ToLower(term)
		if strings.HasPrefix(low, "redirect=") {
			redirect = term[len("redirect="):]
			continue
		}
		if strings.Contains(low, "=") && !strings.ContainsAny(low[:strings.IndexByte(low, '=')], ":/") {
			continue // an unknown modifier such as exp=
		}
		q, mech := "pass", low
		switch low[0] {
		case '+', '-', '~', '?':
			q = map[byte]string{'+': "pass", '-': "fail", '~': "softfail", '?': "neutral"}[low[0]]
			mech = low[1:]
		}
		base, cidr4, cidr6 := mech, -1, -1
		if !strings.HasPrefix(mech, "ip4:") && !strings.HasPrefix(mech, "ip6:") {
			if i := strings.IndexByte(mech, '/'); i >= 0 { // "a/24", "mx:host/24//64"
				var ok bool
				if cidr4, cidr6, ok = parsePrefixes(mech[i+1:]); !ok {
					return "", fmt.Errorf("%w: bad prefix in %q", errSPFPerm, term)
				}
				base = mech[:i]
			}
		}
		name, arg, _ := strings.Cut(base, ":")
		matched := false
		switch name {
		case "all":
			matched = true
		case "ip4", "ip6":
			matched = ipInSpec(e.ip, arg)
		case "a", "mx", "include", "exists", "ptr":
			if name == "ptr" {
				continue // discouraged by the RFC and slow: treated as not matching
			}
			if e.lookups++; e.lookups > spfLookupLimit {
				return "", fmt.Errorf("%w: more than %d DNS lookups", errSPFPerm, spfLookupLimit)
			}
			target := domain
			if arg != "" {
				target = arg
			}
			switch name {
			case "include":
				if arg == "" {
					return "", fmt.Errorf("%w: include without a domain", errSPFPerm)
				}
				res, err := e.check(target, depth+1)
				if err != nil {
					return "", err
				}
				if res == "none" {
					return "", fmt.Errorf("%w: included domain %s has no SPF record", errSPFPerm, target)
				}
				matched = res == "pass"
			case "a":
				matched = e.hostMatches(target, cidr4, cidr6)
			case "exists":
				ips, _ := e.b.lookupIPs(target)
				matched = len(ips) > 0
			case "mx":
				hosts, _ := e.b.lookupMXHosts(target)
				for _, h := range hosts {
					if e.lookups++; e.lookups > spfLookupLimit {
						return "", fmt.Errorf("%w: more than %d DNS lookups", errSPFPerm, spfLookupLimit)
					}
					if e.hostMatches(h, cidr4, cidr6) {
						matched = true
						break
					}
				}
			}
		}
		if matched {
			return q, nil
		}
	}
	if redirect != "" {
		if e.lookups++; e.lookups > spfLookupLimit {
			return "", fmt.Errorf("%w: more than %d DNS lookups", errSPFPerm, spfLookupLimit)
		}
		res, err := e.check(redirect, depth+1)
		if err != nil {
			return "", err
		}
		if res == "none" {
			return "", fmt.Errorf("%w: redirect target %s has no SPF record", errSPFPerm, redirect)
		}
		return res, nil
	}
	return "neutral", nil
}

// parsePrefixes reads "24", "24//64" or "/64" (IPv4 and IPv6 prefix lengths, -1 when absent).
func parsePrefixes(s string) (v4, v6 int, ok bool) {
	v4, v6 = -1, -1
	if strings.HasPrefix(s, "/") && !strings.HasPrefix(s, "//") {
		s = "/" + s // "a//64" arrives here as "/64"
	}
	a, b, has6 := strings.Cut(s, "//")
	if a != "" {
		n, err := strconv.Atoi(a)
		if err != nil || n < 0 || n > 32 {
			return 0, 0, false
		}
		v4 = n
	}
	if has6 {
		n, err := strconv.Atoi(b)
		if err != nil || n < 0 || n > 128 {
			return 0, 0, false
		}
		v6 = n
	}
	return v4, v6, true
}

func (e *spfEval) hostMatches(host string, cidr4, cidr6 int) bool {
	ips, _ := e.b.lookupIPs(host)
	for _, ip := range ips {
		bits := cidr4
		if ip.To4() == nil {
			bits = cidr6
		}
		if bits < 0 {
			if ip.Equal(e.ip) {
				return true
			}
			continue
		}
		if _, n, err := net.ParseCIDR(ip.String() + "/" + strconv.Itoa(bits)); err == nil && n.Contains(e.ip) {
			return true
		}
	}
	return false
}

func ipInSpec(ip net.IP, spec string) bool {
	if strings.Contains(spec, "/") {
		_, n, err := net.ParseCIDR(spec)
		return err == nil && n.Contains(ip)
	}
	p := net.ParseIP(spec)
	return p != nil && p.Equal(ip)
}

// checkSPF says whether the domain's SPF record authorizes the source IP.
func (b *Blacktip) checkSPF(domain, sourceIP string) model.MailFinding {
	ip := net.ParseIP(sourceIP)
	if ip == nil {
		return model.MailFinding{Status: model.FindingWarn, Detail: "the source address isn't an IP address"}
	}
	hint := fmt.Sprintf("Publish a TXT record on %s: \"v=spf1 ip4:%s ~all\". If other services also send as %s, add their include: entries to the same record.", domain, sourceIP, domain)
	if ip.To4() == nil {
		hint = strings.Replace(hint, "ip4:", "ip6:", 1)
	}
	e := &spfEval{b: b, ip: ip}
	res, err := e.check(domain, 0)
	switch {
	case errors.Is(err, errSPFPerm):
		return model.MailFinding{Status: model.FindingFail, Detail: "SPF record is invalid: " + strings.TrimPrefix(err.Error(), "permerror: "),
			Hint: "Receivers treat an invalid SPF record as a failure. Fix the record on " + domain + "; SPF allows at most 10 DNS lookups across include:, a, mx and redirect, so flatten some includes into ip4: entries."}
	case err != nil:
		return model.MailFinding{Status: model.FindingWarn, Detail: "couldn't read the SPF record: " + err.Error()}
	case res == "none":
		return model.MailFinding{Status: model.FindingNone, Detail: "no SPF record published on " + domain, Hint: hint}
	case res == "pass":
		rec, _ := b.spfRecord(domain)
		if low := strings.ToLower(rec); strings.HasSuffix(low, " +all") || strings.HasSuffix(low, " ?all") || strings.HasSuffix(low, " all") {
			return model.MailFinding{Status: model.FindingWarn, Detail: "SPF authorizes " + sourceIP + " but ends in a permissive all, which lets anyone send as " + domain,
				Hint: "End the record with ~all (or -all once you're sure every sender is listed)."}
		}
		return model.MailFinding{Status: model.FindingPass, Detail: "SPF authorizes " + sourceIP}
	default:
		return model.MailFinding{Status: model.FindingFail, Detail: fmt.Sprintf("SPF for %s does not authorize %s (result: %s)", domain, sourceIP, res),
			Hint: fmt.Sprintf("Add ip4:%s to the v=spf1 record on %s, before the final all.", sourceIP, domain)}
	}
}

// ---------- DMARC (RFC 7489) ----------

// dmarcRecord finds the DMARC record that applies to domain: its own, else its
// parent's (a cheap stand-in for the organizational domain, which would need
// the public suffix list).
func (b *Blacktip) dmarcRecord(domain string) (rec, at string, err error) {
	for d := domain; strings.Count(d, ".") >= 1; {
		txts, lerr := b.lookupTXT("_dmarc." + d)
		if lerr != nil && !dnsNotFound(lerr) && len(txts) == 0 {
			return "", "", lerr
		}
		for _, t := range txts {
			if strings.HasPrefix(strings.ToLower(strings.TrimSpace(t)), "v=dmarc1") {
				return strings.TrimSpace(t), d, nil
			}
		}
		_, d, _ = strings.Cut(d, ".")
	}
	return "", "", nil
}

func dmarcTags(rec string) map[string]string {
	tags := map[string]string{}
	for _, f := range strings.Split(rec, ";") {
		if k, v, ok := strings.Cut(strings.TrimSpace(f), "="); ok {
			tags[strings.ToLower(strings.TrimSpace(k))] = strings.TrimSpace(v)
		}
	}
	return tags
}

func (b *Blacktip) checkDMARC(domain string) model.MailFinding {
	rec, at, err := b.dmarcRecord(domain)
	if err != nil {
		return model.MailFinding{Status: model.FindingWarn, Detail: "couldn't read the DMARC record: " + err.Error()}
	}
	if rec == "" {
		return model.MailFinding{Status: model.FindingNone, Detail: "no DMARC record published for " + domain,
			Hint: fmt.Sprintf("Publish a TXT record on _dmarc.%s: \"v=DMARC1; p=none; rua=mailto:dmarc@%s\". Start with p=none, read the reports, then tighten it. Gmail and Yahoo now require a DMARC record from bulk senders.", domain, domain)}
	}
	t := dmarcTags(rec)
	where := ""
	if at != domain {
		where = " (inherited from " + at + ")"
	}
	p := strings.ToLower(t["p"])
	if sp := strings.ToLower(t["sp"]); at != domain && sp != "" {
		p = sp
	}
	switch {
	case p != "none" && p != "quarantine" && p != "reject":
		return model.MailFinding{Status: model.FindingFail, Detail: "DMARC record has no valid p= policy" + where,
			Hint: "Set p=none, p=quarantine or p=reject in the _dmarc." + at + " record; receivers ignore a record without one."}
	case t["rua"] == "":
		return model.MailFinding{Status: model.FindingWarn, Detail: "DMARC policy is " + p + " but there is no rua= address, so you'd get no reports" + where,
			Hint: "Add rua=mailto:dmarc@" + domain + " to the record, then upload the reports you receive on the Mail page."}
	case p == "none":
		return model.MailFinding{Status: model.FindingWarn, Detail: "DMARC is in monitoring mode (p=none)" + where,
			Hint: "Once the reports show only your own legitimate mail passing, move to p=quarantine and then p=reject."}
	}
	return model.MailFinding{Status: model.FindingPass, Detail: "DMARC policy is " + p + where}
}

// ---------- MTA-STS for the sending domain ----------

func (b *Blacktip) checkMTASTS(domain string) model.MailFinding {
	pol := b.stsFor(domain)
	if pol == nil {
		return model.MailFinding{Status: model.FindingNone, Detail: "no MTA-STS policy for " + domain,
			Hint: "Optional but recommended: publish a TXT record on _mta-sts." + domain + " (\"v=STSv1; id=1\") and serve a policy at https://mta-sts." + domain + "/.well-known/mta-sts.txt so senders must use TLS to reach you."}
	}
	switch pol.Mode {
	case "enforce":
		return model.MailFinding{Status: model.FindingPass, Detail: "MTA-STS is enforced"}
	case "testing":
		return model.MailFinding{Status: model.FindingWarn, Detail: "MTA-STS is in testing mode",
			Hint: "Once TLS reports look clean, change mode: testing to mode: enforce in the policy file and bump the id in the DNS record."}
	}
	return model.MailFinding{Status: model.FindingNone, Detail: "MTA-STS policy is switched off (mode: none)"}
}

// ---------- blocklists ----------

func (b *Blacktip) dnsblQuery(name string) ([]string, error) {
	if b.cfg.MailLookupDNSBL != nil {
		return b.cfg.MailLookupDNSBL(name)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
	defer cancel()
	addrs, err := net.DefaultResolver.LookupHost(ctx, name)
	if dnsNotFound(err) {
		return nil, nil
	}
	return addrs, err
}

// reverseIPv4 turns 192.0.2.1 into 1.2.0.192, the form DNSBLs are queried in.
func reverseIPv4(ip net.IP) string {
	v4 := ip.To4()
	if v4 == nil {
		return ""
	}
	return fmt.Sprintf("%d.%d.%d.%d", v4[3], v4[2], v4[1], v4[0])
}

// checkBlocklists asks each DNSBL about the IP. listed are zones that list it;
// answered counts the zones that gave a usable answer (listing refusals for
// open resolvers, answered as 127.255.255.x, don't count as either).
func (b *Blacktip) checkBlocklists(sourceIP string) (listed []string, answered int) {
	rev := reverseIPv4(net.ParseIP(sourceIP))
	if rev == "" {
		return nil, 0 // the common blocklists only cover IPv4
	}
	zones := b.cfg.MailBlocklists
	if zones == nil {
		zones = defaultBlocklists
	}
	var mu sync.Mutex
	var wg sync.WaitGroup
	for _, z := range zones {
		wg.Add(1)
		go func(z string) {
			defer wg.Done()
			addrs, err := b.dnsblQuery(rev + "." + z)
			if err != nil {
				return
			}
			hit, bad := false, false
			for _, a := range addrs {
				if strings.HasPrefix(a, "127.255.255.") {
					bad = true // "query refused": the zone won't answer from this resolver
				} else if strings.HasPrefix(a, "127.") {
					hit = true
				}
			}
			mu.Lock()
			defer mu.Unlock()
			if hit {
				listed = append(listed, z)
			}
			if !bad || hit {
				answered++
			}
		}(z)
	}
	wg.Wait()
	sort.Strings(listed)
	return listed, answered
}

// ---------- putting it together ----------

// sendingDomains are the domains the policy has DKIM keys for: the ones it signs mail as.
func sendingDomains(pol model.MailPolicy) []string {
	seen := map[string]bool{}
	var out []string
	for _, k := range pol.DKIM {
		d := strings.ToLower(k.Domain)
		if d != "" && !seen[d] {
			seen[d] = true
			out = append(out, d)
		}
	}
	sort.Strings(out)
	return out
}

// checkDomains runs the per-domain DNS checks for a policy's sending domains.
func (b *Blacktip) checkDomains(pol model.MailPolicy) map[string]model.DomainCheck {
	doms := sendingDomains(pol)
	if len(doms) == 0 {
		return nil
	}
	out := map[string]model.DomainCheck{}
	var mu sync.Mutex
	var wg sync.WaitGroup
	for _, d := range doms {
		wg.Add(1)
		go func(d string) {
			defer wg.Done()
			c := model.DomainCheck{DMARC: b.checkDMARC(d), MTASTS: b.checkMTASTS(d)}
			if pol.Smarthost != "" {
				c.SPF = model.MailFinding{Status: model.FindingWarn, Detail: "not checked: mail leaves through the smarthost " + pol.Smarthost,
					Hint: "The receiving server sees the smarthost's address, so " + d + "'s SPF record must authorize that provider instead."}
			} else {
				c.SPF = b.checkSPF(d, pol.SourceIP)
			}
			mu.Lock()
			out[d] = c
			mu.Unlock()
		}(d)
	}
	wg.Wait()
	return out
}