Mail deliverability dashboard: findings with fixes, DMARC report ingestion, alerts on regressions

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

deliverability_test.go

@@ -0,0 +1,127 @@
+package hammerhead
+
+import (
+	"strings"
+	"testing"
+
+	"git.veillax.com/Veillax/reef/model"
+)
+
+func find(fs []Finding, id string) *Finding {
+	for i := range fs {
+		if fs[i].ID == id {
+			return &fs[i]
+		}
+	}
+	return nil
+}
+
+func TestFindingsCarryFixes(t *testing.T) {
+	x := &model.MailEgress{ID: "x1", OrgID: "o", SourceIP: "203.0.113.7", HeloName: "mail.example.com"}
+	keys := []*model.DKIMKey{{ID: "k1", OrgID: "o", Domain: "example.com", Selector: "s1"}, {ID: "k2", OrgID: "other", Domain: "x.test", Selector: "s"}}
+	chk := model.MailCheck{
+		PTR: []string{"host.vps.test"}, Port25: "can't connect",
+		DKIM: map[string]string{"k1": "mismatch", "k2": "missing"},
+		Domains: map[string]model.DomainCheck{"example.com": {
+			SPF:    model.MailFinding{Status: model.FindingFail, Detail: "no", Hint: "add ip4"},
+			DMARC:  model.MailFinding{Status: model.FindingNone, Detail: "none", Hint: "publish"},
+			MTASTS: model.MailFinding{Status: model.FindingNone, Detail: "none", Hint: "optional"},
+		}},
+		Blocklists: []string{"zen.spamhaus.org"},
+	}
+	fs := deliverabilityFindings(x, chk, keys)
+	for id, want := range map[string]string{"ptr": "fail", "helo": "fail", "port25": "fail", "dkim:k1": "fail", "spf:example.com": "fail", "blocklist": "fail"} {
+		f := find(fs, id)
+		if f == nil || f.Status != want {
+			t.Errorf("%s: %+v", id, f)
+		} else if want == "fail" && f.Hint == "" {
+			t.Errorf("%s fails without saying how to fix it", id)
+		}
+	}
+	if find(fs, "dkim:k2") != nil {
+		t.Error("another org's key leaked into the checklist")
+	}
+	if f := find(fs, "ptr"); !strings.Contains(f.Detail, "host.vps.test") || !strings.Contains(f.Hint, "mail.example.com") {
+		t.Errorf("ptr: %+v", f)
+	}
+	if find(fs, "mtasts:example.com").alertWorthy() {
+		t.Error("a missing MTA-STS policy is advice, not an alert")
+	}
+	if !find(fs, "dmarc:example.com").alertWorthy() {
+		t.Error("a missing DMARC record should alert")
+	}
+}
+
+func TestFindingsAllGood(t *testing.T) {
+	x := &model.MailEgress{OrgID: "o", SourceIP: "203.0.113.7", HeloName: "mail.example.com"}
+	chk := model.MailCheck{PTRMatchesHelo: true, HeloResolves: true, Port25: "open", Listable: 4,
+		Outcomes: &model.MailOutcomes{Delivered: 100}}
+	for _, f := range deliverabilityFindings(x, chk, nil) {
+		if f.Status != model.FindingPass || f.alertWorthy() {
+			t.Errorf("%+v", f)
+		}
+	}
+}
+
+func TestOutcomeFindings(t *testing.T) {
+	x := &model.MailEgress{OrgID: "o", SourceIP: "203.0.113.7", HeloName: "h"}
+	status := func(o *model.MailOutcomes) string {
+		f := find(deliverabilityFindings(x, model.MailCheck{Outcomes: o}, nil), "outcomes")
+		if f == nil {
+			return ""
+		}
+		return f.Status
+	}
+	for name, c := range map[string]struct {
+		o    *model.MailOutcomes
+		want string
+	}{
+		"too few to judge": {&model.MailOutcomes{Delivered: 2, Failed: 5}, ""},
+		"healthy":          {&model.MailOutcomes{Delivered: 95, Deferred: 5}, "pass"},
+		"some trouble":     {&model.MailOutcomes{Delivered: 80, Failed: 20}, "warn"},
+		"mostly refused":   {&model.MailOutcomes{Delivered: 20, Failed: 80}, "fail"},
+	} {
+		if got := status(c.o); got != c.want {
+			t.Errorf("%s: %q, want %q", name, got, c.want)
+		}
+	}
+	h := outcomeHint(&model.MailOutcomes{Recent: []model.OutcomeFailure{{Domain: "gmail.com", Code: 550, Text: "5.7.26 DMARC authentication failed"}}})
+	if !strings.Contains(h, "gmail.com") || !strings.Contains(h, "SPF, DKIM and DMARC") {
+		t.Errorf("hint: %q", h)
+	}
+	if outcomeHint(&model.MailOutcomes{}) != "" {
+		t.Error("no refusals, no hint")
+	}
+}
+
+func TestDeliverabilityRaisesAlertsWithHints(t *testing.T) {
+	f, _ := dmarcFx(t)
+	f.s.mu.Lock()
+	f.s.mailChecks["x1"] = model.MailCheck{PTRMatchesHelo: true, HeloResolves: true, Port25: "open", Listable: 4,
+		Blocklists: []string{"bl.spamcop.net"},
+		Domains: map[string]model.DomainCheck{"example.com": {
+			SPF:    model.MailFinding{Status: model.FindingFail, Detail: "SPF for example.com does not authorize 203.0.113.7", Hint: "Add ip4:203.0.113.7"},
+			DMARC:  model.MailFinding{Status: model.FindingWarn, Detail: "monitoring"},
+			MTASTS: model.MailFinding{Status: model.FindingNone},
+		}}}
+	probs := f.s.problemsLocked(f.base)
+	f.s.mu.Unlock()
+	spf, bl := probs["mailfind:x1:spf:example.com"], probs["mailfind:x1:blocklist"]
+	if !strings.Contains(spf.msg, "Add ip4:203.0.113.7") || !strings.Contains(bl.msg, "bl.spamcop.net") {
+		t.Fatalf("alerts should carry the fix: %+v %+v", spf, bl)
+	}
+	if _, noisy := probs["mailfind:x1:dmarc:example.com"]; noisy {
+		t.Error("a DMARC warning shouldn't alert")
+	}
+
+	// fixed: both clear
+	f.s.mu.Lock()
+	f.s.mailChecks["x1"] = model.MailCheck{PTRMatchesHelo: true, HeloResolves: true, Port25: "open", Listable: 4}
+	probs = f.s.problemsLocked(f.base)
+	f.s.mu.Unlock()
+	for k := range probs {
+		if strings.HasPrefix(k, "mailfind:") {
+			t.Errorf("%s should have cleared", k)
+		}
+	}
+}