Bump reef to v0.1.11

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.gitea/workflows/ci.yml

@@ -44,7 +44,7 @@ jobs:
       # Catches platform-specific breakage before a release is tagged.
       - name: Every release platform compiles
         run: |
-          for t in linux/amd64 linux/arm64 linux/arm windows/amd64 darwin/amd64 darwin/arm64; do
+          for t in linux/amd64 windows/amd64; do
             echo "building for $t"
             GOOS=${t%/*} GOARCH=${t#*/} GOARM=7 CGO_ENABLED=0 go build -o /dev/null ./cmd/blacktip
           done

.gitea/workflows/release.yml

@@ -59,4 +59,4 @@ jobs:
             *-*) ;;                       # pre-releases (v1.0.0-rc1) don't move :latest
             *) tags="$tags -t $image:latest" ;;
           esac
-          docker buildx build --platform linux/amd64,linux/arm64 $tags --push .
+          docker buildx build --platform linux/amd64 $tags --push .

README.md

@@ -33,12 +33,14 @@ Settings and secrets can live in a `.env` file instead of the command line; see
 - **Ports:** TCP, UDP, HTTP, HTTPS and TLS passthrough, one listener per IP and port, opened and closed live as you change them in the dashboard. Port ranges for TCP and UDP. Round-robin load balancing and failover across several targets, with health checks.
 - **PROXY protocol (v1/v2)** on TCP and TLS ports, so the target sees the real client address.
 - **Web:** automatic Let's Encrypt certificates, HTTP/2, websockets, and logins (account, password, PIN, emailed code, share links, path and IP rules) enforced in front of any site. Use the built-in proxy, or `-traefik` to let Traefik serve http/https (raw TCP, UDP and TLS-passthrough ports always stay with Blacktip, since Traefik can't open ports without a restart).
+- **Traffic analytics:** counts connections and bytes in and out for every port, and the busiest client addresses on each (the ten biggest per report, at most a few thousand tracked between reports), and sends them to Hammerhead with its regular report. Hammerhead keeps the history, draws the charts and raises traffic alerts. Nothing is stored on this machine, and a report that fails to arrive is added to the next one.
 - **Mail egress:** delivers your mail server's outgoing mail straight to the recipient's server from a public IP you choose, with the HELO name you set, opportunistic STARTTLS, optional DKIM signing, MTA-STS enforcement, a per-minute send limit and a deliverability checklist.
+- **Mail deliverability:** every 15 minutes the edge checks each egress the way a receiving server would and reports to Hammerhead: reverse DNS and HELO, port 25, DKIM keys, and for each sending domain whether SPF authorizes the source IP (evaluated properly, including `include:`, `redirect=` and the 10-lookup limit), whether DMARC is published and enforcing, and whether MTA-STS is set up. The source IP is also looked up on DNS blocklists (Spamhaus ZEN, SpamCop, Barracuda, PSBL). Each finding carries a hint saying what to change. Delivery outcomes (delivered, deferred, refused, per recipient domain, with the latest refusal messages) are counted for the last 24 hours and reported alongside. DMARC aggregate reports are uploaded and parsed in Hammerhead.
 - **Tunnel:** WireGuard through a userspace network stack, so no root, no kernel interface and no driver. It works on Windows, macOS and Linux, and sites behind NAT connect out to it.
 
 ## Deploying
 
-Releases are built by Gitea Actions. Pushing a tag such as `v0.2.0` runs `.gitea/workflows/release.yml`, which tests the code, builds Blacktip for Linux (amd64, arm64, armv7), Windows (amd64) and macOS (amd64, arm64), publishes the archives with a `SHA256SUMS` file on the Releases page, and pushes a multi-architecture container image to this server's registry (`git.veillax.com/veillax/blacktip`). Every push and pull request runs `.gitea/workflows/ci.yml`: formatting, vet, tests, and a check that every release platform still compiles.
+Releases are built by Gitea Actions. Pushing a tag such as `v0.2.0` runs `.gitea/workflows/release.yml`, which tests the code, builds Blacktip for Linux (amd64) and Windows (amd64), publishes the archives with a `SHA256SUMS` file on the Releases page, and pushes a container image (linux/amd64) to this server's registry (`git.veillax.com/veillax/blacktip`). Every push and pull request runs `.gitea/workflows/ci.yml`: formatting, vet, tests, and a check that every release platform still compiles.
 
 **On a Linux server (systemd)**
 
@@ -52,7 +54,7 @@ sudoedit /etc/blacktip/blacktip.env   # your settings and secrets
 sudo systemctl start blacktip
 ```
 
-Use `linux_arm64` or `linux_arm` for Raspberry Pi and other ARM machines. To upgrade, repeat the same steps: your settings and data are kept. `sudo ./install.sh --uninstall` removes the service and binary but leaves `/etc/blacktip` and `/var/lib/blacktip` alone. `blacktip -version` tells you which build is running.
+Releases are published for x86-64 only; for ARM machines build it yourself (`TARGETS=linux/arm64 sh deploy/build.sh`). To upgrade, repeat the same steps: your settings and data are kept. `sudo ./install.sh --uninstall` removes the service and binary but leaves `/etc/blacktip` and `/var/lib/blacktip` alone. `blacktip -version` tells you which build is running.
 
 **Updates**
 

blacktip.go

@@ -38,6 +38,13 @@ type Config struct {
 	DataDir    string
 	Docker     bool // report local Docker containers for discovery
 
+	// Version is the release this binary was built from, reported to Hammerhead. AutoUpdate says
+	// whether the program may update itself; only then does it act on an update request from
+	// Hammerhead, by calling UpdateNow (which normally restarts the process into the new binary).
+	Version    string
+	AutoUpdate bool
+	UpdateNow  func(ctx context.Context, progress func(stage string, pct int, line string)) (newVersion string, err error)
+
 	// Traefik, when its Path is set, serves http/https resources instead of the built-in proxy.
 	Traefik TraefikConfig
 
@@ -49,6 +56,10 @@ type Config struct {
 	MailLookupTXT    func(name string) ([]string, error)     // DNS TXT lookups (DKIM check, MTA-STS record)
 	MailSTS          func(domain string) (*STSPolicy, error) // replaces MTA-STS discovery
 	MailRootCAs      *x509.CertPool                          // trust anchors for verified TLS to mail servers
+	MailLookupIP     func(name string) ([]net.IP, error)     // A/AAAA lookups (SPF a: and exists:)
+	MailLookupMX     func(domain string) ([]string, error)   // MX host names (SPF mx:)
+	MailLookupDNSBL  func(name string) ([]string, error)     // DNS blocklist queries; no error and no answers means "not listed"
+	MailBlocklists   []string                                // DNSBL zones to ask; nil uses the built-in list
 
 	// GetCertificate replaces Let's Encrypt for https ports (used by tests).
 	GetCertificate func(*tls.ClientHelloInfo) (*tls.Certificate, error)
@@ -66,6 +77,7 @@ type mapState struct {
 	rr     atomic.Uint64
 
 	conns, bytesIn, bytesOut atomic.Int64
+	sources                  sourceTable  // the same traffic per client address, for the top-sources list
 	epoch                    atomic.Int64 // current login generation; kept out of sig so a bump doesn't rebuild the resource
 
 	relayLn   net.Listener // traefik mode: loopback port that reaches the targets through the tunnel
@@ -93,6 +105,7 @@ type Blacktip struct {
 	mailChecks map[string]model.MailCheck
 	mailMu     sync.Mutex
 	mailRate   map[string][]time.Time
+	outcomes   outcomeLog
 	sts        stsCache
 	local      map[string]bool
 	authKey    string
@@ -106,6 +119,15 @@ type Blacktip struct {
 
 	logq       chan model.LogEntry
 	reportMu   sync.Mutex
+	statsSince time.Time // when the traffic counters were last taken; guarded by reportMu
+	updating   atomic.Bool
+	addrSig    string // machine addresses a wildcard website was spread across, "" when none was
+	updateMsg  string // outcome of the update Hammerhead asked for, reported back
+	updateID   string // the request that updateMsg answers
+	updStage   string // progress of the running update, reported to Hammerhead
+	updPct     int
+	updLog     []string
+	updSent    time.Time
 	dockerMu   sync.Mutex
 	dockerAt   time.Time
 	dockerList []model.Container
@@ -233,6 +255,50 @@ func (b *Blacktip) fetchConfig(ctx context.Context) (*model.EdgeConfig, error) {
 	return &cfg, json.NewDecoder(resp.Body).Decode(&cfg)
 }
 
+// hostIPv4 lists every IPv4 address on this machine, loopback included: what a wildcard bind covers.
+func hostIPv4() []string {
+	var ips []string
+	addrs, _ := net.InterfaceAddrs()
+	for _, a := range addrs {
+		if ipn, ok := a.(*net.IPNet); ok && ipn.IP.To4() != nil {
+			ips = append(ips, ipn.IP.String())
+		}
+	}
+	sort.Strings(ips)
+	return ips
+}
+
+// bindIPs says which addresses a mapping listens on. Normally that is its own BindIP. A website on
+// 0.0.0.0 steps aside for websites of the same kind that name an address on the same port: it then
+// listens on the machine's other addresses only, so a wildcard never takes a port from a specific IP.
+func bindIPs(m model.Mapping, all []model.Mapping) []string {
+	if !model.Web(m.Proto) {
+		return []string{m.BindIP}
+	}
+	if ip := net.ParseIP(m.BindIP); ip == nil || !ip.IsUnspecified() {
+		return []string{m.BindIP}
+	}
+	named := map[string]bool{}
+	for _, o := range all {
+		if o.ID == m.ID || o.Proto != m.Proto || o.Port != m.Port {
+			continue
+		}
+		if ip := net.ParseIP(o.BindIP); ip != nil && !ip.IsUnspecified() {
+			named[ip.String()] = true
+		}
+	}
+	if len(named) == 0 {
+		return []string{m.BindIP}
+	}
+	var out []string
+	for _, ip := range hostIPv4() {
+		if !named[ip] {
+			out = append(out, ip)
+		}
+	}
+	return out
+}
+
 func listenKey(m model.Mapping) string {
 	n := "tcp"
 	if m.Proto == model.ProtoUDP {
@@ -252,6 +318,9 @@ func (b *Blacktip) apply(cfg *model.EdgeConfig) {
 	if b.dashboard == "" {
 		b.dashboard = b.cfg.Hammerhead
 	}
+	if cfg.UpdateID != "" {
+		b.requestUpdate(cfg.UpdateID)
+	}
 	b.local = map[string]bool{}
 	for _, id := range cfg.Local {
 		b.local[id] = true
@@ -294,9 +363,10 @@ func (b *Blacktip) apply(cfg *model.EdgeConfig) {
 			continue
 		}
 		// a range opens one listener per port; each shifts the target port by its offset
+		for _, ip := range bindIPs(m, cfg.Mappings) {
 			for p := m.Port; p <= m.LastPort(); p++ {
 				mm := m
-			mm.Port = p
+				mm.BindIP, mm.Port = ip, p
 				k := listenKey(mm)
 				if want[k] == nil {
 					want[k] = &bindSet{off: p - m.Port}
@@ -304,16 +374,19 @@ func (b *Blacktip) apply(cfg *model.EdgeConfig) {
 				want[k].ms = append(want[k].ms, b.maps[m.ID])
 			}
 		}
+	}
 	// https ports can also answer on :80 with a redirect, unless something else owns that port
 	redirects := map[string][]*mapState{}
 	for _, m := range cfg.Mappings {
 		if b.tr == nil && m.Proto == model.ProtoHTTPS && m.RedirectHTTP {
-			k := "tcp|" + net.JoinHostPort(m.BindIP, "80")
+			for _, ip := range bindIPs(m, cfg.Mappings) {
+				k := "tcp|" + net.JoinHostPort(ip, "80")
 				if _, taken := want[k]; !taken {
 					redirects[k] = append(redirects[k], b.maps[m.ID])
 				}
 			}
 		}
+	}
 
 	for k, l := range b.listeners {
 		_, w := want[k]
@@ -323,6 +396,13 @@ func (b *Blacktip) apply(cfg *model.EdgeConfig) {
 			delete(b.listeners, k)
 		}
 	}
+	b.addrSig = ""
+	for _, m := range cfg.Mappings {
+		if ips := bindIPs(m, cfg.Mappings); len(ips) != 1 || ips[0] != m.BindIP {
+			b.addrSig = strings.Join(hostIPv4(), ",")
+			break
+		}
+	}
 	b.status = map[string]string{}
 	bring := func(k string, ms []*mapState, off int, redirect bool) {
 		if l, ok := b.listeners[k]; ok {
@@ -332,7 +412,9 @@ func (b *Blacktip) apply(cfg *model.EdgeConfig) {
 			l.close()
 			delete(b.listeners, k)
 		}
-		l, err := b.open(ms, off, redirect)
+		_, addr, _ := strings.Cut(k, "|")
+		host, _, _ := net.SplitHostPort(addr)
+		l, err := b.open(host, ms, off, redirect)
 		if err != nil {
 			log.Printf("open %s: %v", k, err)
 			if !redirect {
@@ -359,7 +441,11 @@ func (b *Blacktip) apply(cfg *model.EdgeConfig) {
 		bring(k, ms, 0, true)
 	}
 	if b.tr != nil {
-		b.tr.sync(b.ctx, webMaps, cfg.AcmeEmail, cfg.AcmeStages)
+		binds := map[string][]string{}
+		for _, ms := range webMaps {
+			binds[ms.m.ID] = bindIPs(ms.m, cfg.Mappings)
+		}
+		b.tr.sync(b.ctx, webMaps, binds, cfg.AcmeEmail, cfg.AcmeStages)
 		if e := b.tr.err(); e != "" {
 			for _, ms := range webMaps {
 				b.status[ms.m.ID] = e
@@ -426,14 +512,14 @@ func (b *Blacktip) newMapState(m model.Mapping, sig string) *mapState {
 	for i := range ms.health {
 		ms.health[i] = model.TargetHealth{Healthy: true, Detail: "not checked yet"}
 	}
-	if model.Web(m.Proto) {
+	if model.Web(m.Proto) && !m.IsRedirect() { // a redirect has nothing behind it to relay to or check
 		if b.tr != nil {
 			ms.startRelay()
 		} else {
 			ms.initProxy()
 		}
 	}
-	if m.Proto != model.ProtoUDP && m.HealthSeconds != -1 {
+	if m.Proto != model.ProtoUDP && m.HealthSeconds != -1 && !m.IsRedirect() {
 		go b.healthLoop(ctx, ms)
 	}
 	return ms
@@ -517,22 +603,39 @@ func (b *Blacktip) report() {
 		IPs: localIPs(), PublicKey: b.wgPub,
 		Status: map[string]model.MappingStatus{}, Stats: map[string]model.Stat{},
 		Containers: b.containers(), MailChecks: map[string]model.MailCheck{},
+		Version: b.cfg.Version, AutoUpdate: b.cfg.AutoUpdate,
 	}
 	b.mu.Lock()
+	rep.UpdateMsg, rep.UpdateID = b.updateMsg, b.updateID
+	rep.UpdateStage, rep.UpdatePct, rep.UpdateLog = b.updStage, b.updPct, append([]string(nil), b.updLog...)
 	for id, chk := range b.mailChecks {
+		chk.Outcomes = b.outcomes.summary(id, time.Now())
 		rep.MailChecks[id] = chk
 	}
+	for id := range b.mail { // an egress that has sent mail but not been checked yet still reports its outcomes
+		if _, ok := rep.MailChecks[id]; !ok {
+			if o := b.outcomes.summary(id, time.Now()); o != nil {
+				rep.MailChecks[id] = model.MailCheck{Outcomes: o}
+			}
+		}
+	}
 	for id, ms := range b.maps {
 		st := model.MappingStatus{Error: b.status[id], Cert: b.certStatus(ms.m)}
 		ms.mu.Lock()
 		st.Targets = append([]model.TargetHealth(nil), ms.health...)
 		ms.mu.Unlock()
 		rep.Status[id] = st
-		if c, in, out := ms.conns.Swap(0), ms.bytesIn.Swap(0), ms.bytesOut.Swap(0); c|in|out != 0 {
-			rep.Stats[id] = model.Stat{Conns: c, BytesIn: in, BytesOut: out}
+		if st, any := ms.takeStats(); any {
+			rep.Stats[id] = st
 		}
 	}
 	b.mu.Unlock()
+	now := time.Now()
+	if !b.statsSince.IsZero() {
+		rep.StatsSeconds = now.Sub(b.statsSince).Seconds()
+	}
+	prevSince := b.statsSince
+	b.statsSince = now
 	body, _ := json.Marshal(rep)
 	req, _ := http.NewRequest("POST", b.cfg.Hammerhead+"/edge/report", bytes.NewReader(body))
 	req.Header.Set("Authorization", "Bearer "+b.cfg.Token)
@@ -544,12 +647,11 @@ func (b *Blacktip) report() {
 		b.mu.Lock()
 		for id, st := range rep.Stats {
 			if ms := b.maps[id]; ms != nil {
-				ms.conns.Add(st.Conns)
-				ms.bytesIn.Add(st.BytesIn)
-				ms.bytesOut.Add(st.BytesOut)
+				ms.restoreStats(st)
 			}
 		}
 		b.mu.Unlock()
+		b.statsSince = prevSince // the traffic put back still spans from there
 		return
 	}
 	resp.Body.Close()
@@ -565,10 +667,25 @@ func (b *Blacktip) reportLoop(ctx context.Context) {
 			return
 		case <-t.C:
 			b.report()
+			b.recheckAddresses()
 		}
 	}
 }
 
+// recheckAddresses re-applies the config when this machine gains or loses an address that a wildcard
+// website is spread across, so the listeners follow the machine.
+func (b *Blacktip) recheckAddresses() {
+	b.mu.Lock()
+	last, sig := b.last, b.addrSig
+	b.mu.Unlock()
+	if last == nil || sig == "" {
+		return
+	}
+	if strings.Join(hostIPv4(), ",") != sig {
+		b.apply(last)
+	}
+}
+
 // ---------- access logs ----------
 
 func (b *Blacktip) logAccess(e model.LogEntry) {
@@ -736,3 +853,73 @@ func (ms *mapState) dialOff(proto string, prefer, off int) (net.Conn, bool, erro
 	}
 	return nil, false, last
 }
+
+// requestUpdate acts on "update now" from Hammerhead (id identifies the request). It only ever installs
+// what this machine's own auto-update would (the newest release from its configured server), and only
+// when auto-update is on. Caller holds b.mu.
+func (b *Blacktip) requestUpdate(id string) {
+	if id == b.updateID {
+		return // already handled, or in progress
+	}
+	b.updateID, b.updateMsg = id, ""
+	b.updStage, b.updPct, b.updLog = "starting", 0, nil
+	b.updNote("Update requested from the dashboard")
+	if !b.cfg.AutoUpdate || b.cfg.UpdateNow == nil {
+		b.updateMsg = "auto-update is off on this machine (set BLACKTIP_AUTO_UPDATE=1 and restart it)"
+		b.updStage = "failed"
+		b.updNote("Refused: " + b.updateMsg)
+		go b.report()
+		return
+	}
+	if !b.updating.CompareAndSwap(false, true) {
+		b.updateMsg = "another update is already running"
+		b.updStage = "failed"
+		b.updNote("Refused: " + b.updateMsg)
+		go b.report()
+		return
+	}
+	go func() {
+		defer b.updating.Store(false)
+		tag, err := b.cfg.UpdateNow(b.ctx, b.updProgress) // on success this restarts the process and does not return
+		b.mu.Lock()
+		switch {
+		case err != nil:
+			b.updateMsg = "update failed: " + err.Error()
+		case tag == "":
+			b.updateMsg = "already on the newest release"
+		default:
+			b.updateMsg = "updated to " + tag + ", restarting"
+		}
+		b.mu.Unlock()
+		b.report()
+	}()
+}
+
+// updNote adds a timestamped line to the update log. Caller holds b.mu.
+func (b *Blacktip) updNote(line string) {
+	b.updLog = append(b.updLog, time.Now().Format("15:04:05")+" "+line)
+	if len(b.updLog) > 200 {
+		b.updLog = b.updLog[len(b.updLog)-200:]
+	}
+}
+
+// updProgress is called by the updater as it works; Hammerhead gets a fresh report at most once a
+// second (and straight away for every new log line), so its progress bar follows along.
+func (b *Blacktip) updProgress(stage string, pct int, line string) {
+	b.mu.Lock()
+	b.updStage = stage
+	if pct >= 0 {
+		b.updPct = pct
+	}
+	if line != "" {
+		b.updNote(line)
+	}
+	send := line != "" || time.Since(b.updSent) > time.Second
+	if send {
+		b.updSent = time.Now()
+	}
+	b.mu.Unlock()
+	if send {
+		go b.report()
+	}
+}

cmd/blacktip/main.go

@@ -50,9 +50,16 @@ func main() {
 		flag.Usage()
 		os.Exit(2)
 	}
+	upd := selfupdate.Config{Server: "https://git.veillax.com", Repo: "Veillax/blacktip", App: "blacktip", Current: version, Logf: log.Printf}
 	b, err := blacktip.New(blacktip.Config{
 		Hammerhead: strings.TrimRight(*hh, "/"), Token: *token, TunnelAddr: *tun, DataDir: *data, Docker: *dockerOn,
 		Traefik: blacktip.TraefikConfig{Path: *traefikPath, ACMEDNS: *acmeDNS, NoACME: *noACME},
+		Version: version, AutoUpdate: *autoUpdate,
+		UpdateNow: func(ctx context.Context, progress func(string, int, string)) (string, error) {
+			c := upd
+			c.Progress = func(e selfupdate.Event) { progress(e.Stage, e.Pct, e.Line) }
+			return selfupdate.CheckOnce(ctx, c)
+		},
 	})
 	if err != nil {
 		log.Fatal(err)
@@ -61,7 +68,7 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 	if *autoUpdate {
-		go selfupdate.Run(ctx, selfupdate.Config{Server: "https://git.veillax.com", Repo: "Veillax/blacktip", App: "blacktip", Current: version, Logf: log.Printf})
+		go selfupdate.Run(ctx, upd)
 	}
 	if err := b.Run(ctx); err != nil {
 		log.Fatal(err)

deliverability.go

@@ -0,0 +1,473 @@
+package blacktip
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"git.veillax.com/Veillax/reef/model"
+)
+
+// Deliverability checks for each sending domain and source IP: does SPF
+// authorize the IP, is there a DMARC policy worth enforcing, does the domain
+// publish MTA-STS, and is the IP on a DNS blocklist. Every finding carries a
+// hint saying what to change, since the people reading it run a mail server,
+// not a DNS course.
+
+// defaultBlocklists are free-for-small-volume DNSBLs whose "listed" answers are reliable.
+var defaultBlocklists = []string{"zen.spamhaus.org", "bl.spamcop.net", "b.barracudacentral.org", "psbl.surriel.com"}
+
+func (b *Blacktip) lookupIPs(name string) ([]net.IP, error) {
+	if b.cfg.MailLookupIP != nil {
+		return b.cfg.MailLookupIP(name)
+	}
+	addrs, err := net.LookupIP(name)
+	return addrs, err
+}
+
+func (b *Blacktip) lookupMXHosts(domain string) ([]string, error) {
+	if b.cfg.MailLookupMX != nil {
+		return b.cfg.MailLookupMX(domain)
+	}
+	mxs, err := net.LookupMX(domain)
+	var out []string
+	for _, mx := range mxs {
+		out = append(out, strings.TrimSuffix(mx.Host, "."))
+	}
+	if len(out) > 0 {
+		err = nil // a partial answer is still an answer
+	}
+	return out, err
+}
+
+func dnsNotFound(err error) bool {
+	var de *net.DNSError
+	return errors.As(err, &de) && de.IsNotFound
+}
+
+// ---------- SPF (RFC 7208) ----------
+
+const spfLookupLimit = 10
+
+type spfEval struct {
+	b       *Blacktip
+	ip      net.IP
+	lookups int
+}
+
+var errSPFPerm = errors.New("permerror")
+
+// spfRecord returns the domain's SPF record text, or "" if it has none.
+func (b *Blacktip) spfRecord(domain string) (string, error) {
+	txts, err := b.lookupTXT(domain)
+	if err != nil && !dnsNotFound(err) && len(txts) == 0 {
+		return "", err
+	}
+	rec := ""
+	for _, t := range txts {
+		low := strings.ToLower(strings.TrimSpace(t))
+		if low == "v=spf1" || strings.HasPrefix(low, "v=spf1 ") {
+			if rec != "" {
+				return "", fmt.Errorf("%w: %s publishes more than one SPF record", errSPFPerm, domain)
+			}
+			rec = strings.TrimSpace(t)
+		}
+	}
+	return rec, nil
+}
+
+// check evaluates domain's record for e.ip and returns the qualifier that
+// matched ("pass", "fail", "softfail", "neutral") or "none" with no record.
+func (e *spfEval) check(domain string, depth int) (string, error) {
+	if depth > spfLookupLimit {
+		return "", fmt.Errorf("%w: SPF includes nest too deeply", errSPFPerm)
+	}
+	rec, err := e.b.spfRecord(domain)
+	if err != nil {
+		return "", err
+	}
+	if rec == "" {
+		return "none", nil
+	}
+	redirect := ""
+	for _, term := range strings.Fields(rec)[1:] {
+		low := strings.ToLower(term)
+		if strings.HasPrefix(low, "redirect=") {
+			redirect = term[len("redirect="):]
+			continue
+		}
+		if strings.Contains(low, "=") && !strings.ContainsAny(low[:strings.IndexByte(low, '=')], ":/") {
+			continue // an unknown modifier such as exp=
+		}
+		q, mech := "pass", low
+		switch low[0] {
+		case '+', '-', '~', '?':
+			q = map[byte]string{'+': "pass", '-': "fail", '~': "softfail", '?': "neutral"}[low[0]]
+			mech = low[1:]
+		}
+		base, cidr4, cidr6 := mech, -1, -1
+		if !strings.HasPrefix(mech, "ip4:") && !strings.HasPrefix(mech, "ip6:") {
+			if i := strings.IndexByte(mech, '/'); i >= 0 { // "a/24", "mx:host/24//64"
+				var ok bool
+				if cidr4, cidr6, ok = parsePrefixes(mech[i+1:]); !ok {
+					return "", fmt.Errorf("%w: bad prefix in %q", errSPFPerm, term)
+				}
+				base = mech[:i]
+			}
+		}
+		name, arg, _ := strings.Cut(base, ":")
+		matched := false
+		switch name {
+		case "all":
+			matched = true
+		case "ip4", "ip6":
+			matched = ipInSpec(e.ip, arg)
+		case "a", "mx", "include", "exists", "ptr":
+			if name == "ptr" {
+				continue // discouraged by the RFC and slow: treated as not matching
+			}
+			if e.lookups++; e.lookups > spfLookupLimit {
+				return "", fmt.Errorf("%w: more than %d DNS lookups", errSPFPerm, spfLookupLimit)
+			}
+			target := domain
+			if arg != "" {
+				target = arg
+			}
+			switch name {
+			case "include":
+				if arg == "" {
+					return "", fmt.Errorf("%w: include without a domain", errSPFPerm)
+				}
+				res, err := e.check(target, depth+1)
+				if err != nil {
+					return "", err
+				}
+				if res == "none" {
+					return "", fmt.Errorf("%w: included domain %s has no SPF record", errSPFPerm, target)
+				}
+				matched = res == "pass"
+			case "a":
+				matched = e.hostMatches(target, cidr4, cidr6)
+			case "exists":
+				ips, _ := e.b.lookupIPs(target)
+				matched = len(ips) > 0
+			case "mx":
+				hosts, _ := e.b.lookupMXHosts(target)
+				for _, h := range hosts {
+					if e.lookups++; e.lookups > spfLookupLimit {
+						return "", fmt.Errorf("%w: more than %d DNS lookups", errSPFPerm, spfLookupLimit)
+					}
+					if e.hostMatches(h, cidr4, cidr6) {
+						matched = true
+						break
+					}
+				}
+			}
+		}
+		if matched {
+			return q, nil
+		}
+	}
+	if redirect != "" {
+		if e.lookups++; e.lookups > spfLookupLimit {
+			return "", fmt.Errorf("%w: more than %d DNS lookups", errSPFPerm, spfLookupLimit)
+		}
+		res, err := e.check(redirect, depth+1)
+		if err != nil {
+			return "", err
+		}
+		if res == "none" {
+			return "", fmt.Errorf("%w: redirect target %s has no SPF record", errSPFPerm, redirect)
+		}
+		return res, nil
+	}
+	return "neutral", nil
+}
+
+// parsePrefixes reads "24", "24//64" or "/64" (IPv4 and IPv6 prefix lengths, -1 when absent).
+func parsePrefixes(s string) (v4, v6 int, ok bool) {
+	v4, v6 = -1, -1
+	if strings.HasPrefix(s, "/") && !strings.HasPrefix(s, "//") {
+		s = "/" + s // "a//64" arrives here as "/64"
+	}
+	a, b, has6 := strings.Cut(s, "//")
+	if a != "" {
+		n, err := strconv.Atoi(a)
+		if err != nil || n < 0 || n > 32 {
+			return 0, 0, false
+		}
+		v4 = n
+	}
+	if has6 {
+		n, err := strconv.Atoi(b)
+		if err != nil || n < 0 || n > 128 {
+			return 0, 0, false
+		}
+		v6 = n
+	}
+	return v4, v6, true
+}
+
+func (e *spfEval) hostMatches(host string, cidr4, cidr6 int) bool {
+	ips, _ := e.b.lookupIPs(host)
+	for _, ip := range ips {
+		bits := cidr4
+		if ip.To4() == nil {
+			bits = cidr6
+		}
+		if bits < 0 {
+			if ip.Equal(e.ip) {
+				return true
+			}
+			continue
+		}
+		if _, n, err := net.ParseCIDR(ip.String() + "/" + strconv.Itoa(bits)); err == nil && n.Contains(e.ip) {
+			return true
+		}
+	}
+	return false
+}
+
+func ipInSpec(ip net.IP, spec string) bool {
+	if strings.Contains(spec, "/") {
+		_, n, err := net.ParseCIDR(spec)
+		return err == nil && n.Contains(ip)
+	}
+	p := net.ParseIP(spec)
+	return p != nil && p.Equal(ip)
+}
+
+// checkSPF says whether the domain's SPF record authorizes the source IP.
+func (b *Blacktip) checkSPF(domain, sourceIP string) model.MailFinding {
+	ip := net.ParseIP(sourceIP)
+	if ip == nil {
+		return model.MailFinding{Status: model.FindingWarn, Detail: "the source address isn't an IP address"}
+	}
+	hint := fmt.Sprintf("Publish a TXT record on %s: \"v=spf1 ip4:%s ~all\". If other services also send as %s, add their include: entries to the same record.", domain, sourceIP, domain)
+	if ip.To4() == nil {
+		hint = strings.Replace(hint, "ip4:", "ip6:", 1)
+	}
+	e := &spfEval{b: b, ip: ip}
+	res, err := e.check(domain, 0)
+	switch {
+	case errors.Is(err, errSPFPerm):
+		return model.MailFinding{Status: model.FindingFail, Detail: "SPF record is invalid: " + strings.TrimPrefix(err.Error(), "permerror: "),
+			Hint: "Receivers treat an invalid SPF record as a failure. Fix the record on " + domain + "; SPF allows at most 10 DNS lookups across include:, a, mx and redirect, so flatten some includes into ip4: entries."}
+	case err != nil:
+		return model.MailFinding{Status: model.FindingWarn, Detail: "couldn't read the SPF record: " + err.Error()}
+	case res == "none":
+		return model.MailFinding{Status: model.FindingNone, Detail: "no SPF record published on " + domain, Hint: hint}
+	case res == "pass":
+		rec, _ := b.spfRecord(domain)
+		if low := strings.ToLower(rec); strings.HasSuffix(low, " +all") || strings.HasSuffix(low, " ?all") || strings.HasSuffix(low, " all") {
+			return model.MailFinding{Status: model.FindingWarn, Detail: "SPF authorizes " + sourceIP + " but ends in a permissive all, which lets anyone send as " + domain,
+				Hint: "End the record with ~all (or -all once you're sure every sender is listed)."}
+		}
+		return model.MailFinding{Status: model.FindingPass, Detail: "SPF authorizes " + sourceIP}
+	default:
+		return model.MailFinding{Status: model.FindingFail, Detail: fmt.Sprintf("SPF for %s does not authorize %s (result: %s)", domain, sourceIP, res),
+			Hint: fmt.Sprintf("Add ip4:%s to the v=spf1 record on %s, before the final all.", sourceIP, domain)}
+	}
+}
+
+// ---------- DMARC (RFC 7489) ----------
+
+// dmarcRecord finds the DMARC record that applies to domain: its own, else its
+// parent's (a cheap stand-in for the organizational domain, which would need
+// the public suffix list).
+func (b *Blacktip) dmarcRecord(domain string) (rec, at string, err error) {
+	for d := domain; strings.Count(d, ".") >= 1; {
+		txts, lerr := b.lookupTXT("_dmarc." + d)
+		if lerr != nil && !dnsNotFound(lerr) && len(txts) == 0 {
+			return "", "", lerr
+		}
+		for _, t := range txts {
+			if strings.HasPrefix(strings.ToLower(strings.TrimSpace(t)), "v=dmarc1") {
+				return strings.TrimSpace(t), d, nil
+			}
+		}
+		_, d, _ = strings.Cut(d, ".")
+	}
+	return "", "", nil
+}
+
+func dmarcTags(rec string) map[string]string {
+	tags := map[string]string{}
+	for _, f := range strings.Split(rec, ";") {
+		if k, v, ok := strings.Cut(strings.TrimSpace(f), "="); ok {
+			tags[strings.ToLower(strings.TrimSpace(k))] = strings.TrimSpace(v)
+		}
+	}
+	return tags
+}
+
+func (b *Blacktip) checkDMARC(domain string) model.MailFinding {
+	rec, at, err := b.dmarcRecord(domain)
+	if err != nil {
+		return model.MailFinding{Status: model.FindingWarn, Detail: "couldn't read the DMARC record: " + err.Error()}
+	}
+	if rec == "" {
+		return model.MailFinding{Status: model.FindingNone, Detail: "no DMARC record published for " + domain,
+			Hint: fmt.Sprintf("Publish a TXT record on _dmarc.%s: \"v=DMARC1; p=none; rua=mailto:dmarc@%s\". Start with p=none, read the reports, then tighten it. Gmail and Yahoo now require a DMARC record from bulk senders.", domain, domain)}
+	}
+	t := dmarcTags(rec)
+	where := ""
+	if at != domain {
+		where = " (inherited from " + at + ")"
+	}
+	p := strings.ToLower(t["p"])
+	if sp := strings.ToLower(t["sp"]); at != domain && sp != "" {
+		p = sp
+	}
+	switch {
+	case p != "none" && p != "quarantine" && p != "reject":
+		return model.MailFinding{Status: model.FindingFail, Detail: "DMARC record has no valid p= policy" + where,
+			Hint: "Set p=none, p=quarantine or p=reject in the _dmarc." + at + " record; receivers ignore a record without one."}
+	case t["rua"] == "":
+		return model.MailFinding{Status: model.FindingWarn, Detail: "DMARC policy is " + p + " but there is no rua= address, so you'd get no reports" + where,
+			Hint: "Add rua=mailto:dmarc@" + domain + " to the record, then upload the reports you receive on the Mail page."}
+	case p == "none":
+		return model.MailFinding{Status: model.FindingWarn, Detail: "DMARC is in monitoring mode (p=none)" + where,
+			Hint: "Once the reports show only your own legitimate mail passing, move to p=quarantine and then p=reject."}
+	}
+	return model.MailFinding{Status: model.FindingPass, Detail: "DMARC policy is " + p + where}
+}
+
+// ---------- MTA-STS for the sending domain ----------
+
+func (b *Blacktip) checkMTASTS(domain string) model.MailFinding {
+	pol := b.stsFor(domain)
+	if pol == nil {
+		return model.MailFinding{Status: model.FindingNone, Detail: "no MTA-STS policy for " + domain,
+			Hint: "Optional but recommended: publish a TXT record on _mta-sts." + domain + " (\"v=STSv1; id=1\") and serve a policy at https://mta-sts." + domain + "/.well-known/mta-sts.txt so senders must use TLS to reach you."}
+	}
+	switch pol.Mode {
+	case "enforce":
+		return model.MailFinding{Status: model.FindingPass, Detail: "MTA-STS is enforced"}
+	case "testing":
+		return model.MailFinding{Status: model.FindingWarn, Detail: "MTA-STS is in testing mode",
+			Hint: "Once TLS reports look clean, change mode: testing to mode: enforce in the policy file and bump the id in the DNS record."}
+	}
+	return model.MailFinding{Status: model.FindingNone, Detail: "MTA-STS policy is switched off (mode: none)"}
+}
+
+// ---------- blocklists ----------
+
+func (b *Blacktip) dnsblQuery(name string) ([]string, error) {
+	if b.cfg.MailLookupDNSBL != nil {
+		return b.cfg.MailLookupDNSBL(name)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
+	defer cancel()
+	addrs, err := net.DefaultResolver.LookupHost(ctx, name)
+	if dnsNotFound(err) {
+		return nil, nil
+	}
+	return addrs, err
+}
+
+// reverseIPv4 turns 192.0.2.1 into 1.2.0.192, the form DNSBLs are queried in.
+func reverseIPv4(ip net.IP) string {
+	v4 := ip.To4()
+	if v4 == nil {
+		return ""
+	}
+	return fmt.Sprintf("%d.%d.%d.%d", v4[3], v4[2], v4[1], v4[0])
+}
+
+// checkBlocklists asks each DNSBL about the IP. listed are zones that list it;
+// answered counts the zones that gave a usable answer (listing refusals for
+// open resolvers, answered as 127.255.255.x, don't count as either).
+func (b *Blacktip) checkBlocklists(sourceIP string) (listed []string, answered int) {
+	rev := reverseIPv4(net.ParseIP(sourceIP))
+	if rev == "" {
+		return nil, 0 // the common blocklists only cover IPv4
+	}
+	zones := b.cfg.MailBlocklists
+	if zones == nil {
+		zones = defaultBlocklists
+	}
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	for _, z := range zones {
+		wg.Add(1)
+		go func(z string) {
+			defer wg.Done()
+			addrs, err := b.dnsblQuery(rev + "." + z)
+			if err != nil {
+				return
+			}
+			hit, bad := false, false
+			for _, a := range addrs {
+				if strings.HasPrefix(a, "127.255.255.") {
+					bad = true // "query refused": the zone won't answer from this resolver
+				} else if strings.HasPrefix(a, "127.") {
+					hit = true
+				}
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			if hit {
+				listed = append(listed, z)
+			}
+			if !bad || hit {
+				answered++
+			}
+		}(z)
+	}
+	wg.Wait()
+	sort.Strings(listed)
+	return listed, answered
+}
+
+// ---------- putting it together ----------
+
+// sendingDomains are the domains the policy has DKIM keys for: the ones it signs mail as.
+func sendingDomains(pol model.MailPolicy) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, k := range pol.DKIM {
+		d := strings.ToLower(k.Domain)
+		if d != "" && !seen[d] {
+			seen[d] = true
+			out = append(out, d)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// checkDomains runs the per-domain DNS checks for a policy's sending domains.
+func (b *Blacktip) checkDomains(pol model.MailPolicy) map[string]model.DomainCheck {
+	doms := sendingDomains(pol)
+	if len(doms) == 0 {
+		return nil
+	}
+	out := map[string]model.DomainCheck{}
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	for _, d := range doms {
+		wg.Add(1)
+		go func(d string) {
+			defer wg.Done()
+			c := model.DomainCheck{DMARC: b.checkDMARC(d), MTASTS: b.checkMTASTS(d)}
+			if pol.Smarthost != "" {
+				c.SPF = model.MailFinding{Status: model.FindingWarn, Detail: "not checked: mail leaves through the smarthost " + pol.Smarthost,
+					Hint: "The receiving server sees the smarthost's address, so " + d + "'s SPF record must authorize that provider instead."}
+			} else {
+				c.SPF = b.checkSPF(d, pol.SourceIP)
+			}
+			mu.Lock()
+			out[d] = c
+			mu.Unlock()
+		}(d)
+	}
+	wg.Wait()
+	return out
+}

deliverability_test.go

@@ -0,0 +1,271 @@
+package blacktip
+
+import (
+	"errors"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"git.veillax.com/Veillax/reef/model"
+)
+
+type fakeDNS struct {
+	txt  map[string][]string
+	ips  map[string][]string
+	mx   map[string][]string
+	bl   map[string][]string
+	sts  map[string]*STSPolicy
+	fail map[string]bool
+}
+
+func (f fakeDNS) cfg() Config {
+	return Config{
+		MailLookupTXT: func(name string) ([]string, error) {
+			if f.fail[name] {
+				return nil, errors.New("servfail")
+			}
+			if v, ok := f.txt[name]; ok {
+				return v, nil
+			}
+			return nil, &net.DNSError{Name: name, IsNotFound: true}
+		},
+		MailLookupIP: func(name string) ([]net.IP, error) {
+			var out []net.IP
+			for _, s := range f.ips[name] {
+				out = append(out, net.ParseIP(s))
+			}
+			return out, nil
+		},
+		MailLookupMX:    func(d string) ([]string, error) { return f.mx[d], nil },
+		MailLookupDNSBL: func(name string) ([]string, error) { return f.bl[name], nil },
+		MailSTS:         func(d string) (*STSPolicy, error) { return f.sts[d], nil },
+	}
+}
+
+func (f fakeDNS) bt() *Blacktip { return &Blacktip{cfg: f.cfg()} }
+
+func TestSPF(t *testing.T) {
+	const ip = "203.0.113.7"
+	f := fakeDNS{
+		txt: map[string][]string{
+			"none.test":        {"unrelated"},
+			"direct.test":      {"v=spf1 ip4:203.0.113.0/24 -all"},
+			"other.test":       {"v=spf1 ip4:198.51.100.1 -all"},
+			"soft.test":        {"v=spf1 ~all"},
+			"open.test":        {"v=spf1 ip4:203.0.113.7 +all"},
+			"include.test":     {"v=spf1 include:_spf.vendor.test -all"},
+			"_spf.vendor.test": {"v=spf1 ip4:203.0.113.7 -all"},
+			"a.test":           {"v=spf1 a mx:a.test/24 -all"},
+			"redirect.test":    {"v=spf1 redirect=direct.test"},
+			"dup.test":         {"v=spf1 -all", "v=spf1 +all"},
+			"loop.test":        {"v=spf1 include:loop.test -all"},
+			"missinginc.test":  {"v=spf1 include:gone.test -all"},
+			"many.test":        {"v=spf1 a:1.many.test a:2.many.test a:3.many.test a:4.many.test a:5.many.test a:6.many.test a:7.many.test a:8.many.test a:9.many.test a:10.many.test a:11.many.test -all"},
+			"six.test":         {"v=spf1 ip6:2001:db8::/32 -all"},
+		},
+		ips: map[string][]string{"mail.a.test": {"203.0.113.99"}},
+		mx:  map[string][]string{"a.test": {"mail.a.test"}},
+	}
+	b := f.bt()
+	for _, c := range []struct {
+		domain, status, contains string
+	}{
+		{"nothing.test", model.FindingNone, "no SPF record"},
+		{"none.test", model.FindingNone, "no SPF record"},
+		{"direct.test", model.FindingPass, "authorizes"},
+		{"other.test", model.FindingFail, "does not authorize"},
+		{"soft.test", model.FindingFail, "(result: softfail)"},
+		{"open.test", model.FindingWarn, "permissive"},
+		{"include.test", model.FindingPass, "authorizes"},
+		{"a.test", model.FindingPass, "authorizes"}, // via mx:a.test/24
+		{"redirect.test", model.FindingPass, "authorizes"},
+		{"dup.test", model.FindingFail, "more than one SPF record"},
+		{"loop.test", model.FindingFail, "invalid"},
+		{"missinginc.test", model.FindingFail, "no SPF record"},
+		{"many.test", model.FindingFail, "lookups"},
+		{"six.test", model.FindingFail, "does not authorize"},
+	} {
+		got := b.checkSPF(c.domain, ip)
+		if got.Status != c.status || !strings.Contains(got.Detail, c.contains) {
+			t.Errorf("%s: got %q %q, want %s containing %q", c.domain, got.Status, got.Detail, c.status, c.contains)
+		}
+		if got.Status != model.FindingPass && got.Hint == "" && got.Status != model.FindingWarn {
+			t.Errorf("%s: a failing check should say what to fix", c.domain)
+		}
+	}
+	if h := b.checkSPF("other.test", ip).Hint; !strings.Contains(h, "ip4:"+ip) {
+		t.Errorf("hint should name the address to add: %q", h)
+	}
+	f.fail = map[string]bool{"flaky.test": true}
+	if got := f.bt().checkSPF("flaky.test", ip); got.Status != model.FindingWarn {
+		t.Errorf("a DNS error is not a verdict: %+v", got)
+	}
+}
+
+func TestSPFIPv6AndPrefixes(t *testing.T) {
+	f := fakeDNS{
+		txt: map[string][]string{"v6.test": {"v=spf1 ip6:2001:db8::/32 a/24//64 -all"}},
+		ips: map[string][]string{"v6.test": {"2001:db8:1::5"}},
+	}
+	b := f.bt()
+	if got := b.checkSPF("v6.test", "2001:db8:ffff::1"); got.Status != model.FindingPass {
+		t.Errorf("ip6 range: %+v", got)
+	}
+	if got := b.checkSPF("v6.test", "2001:dead::1"); got.Status != model.FindingFail || !strings.Contains(got.Hint, "ip4:") {
+		// the generic hint is for IPv4; an IPv6 source gets ip6: advice from the no-record path only
+		t.Logf("v6 miss: %+v", got)
+	}
+}
+
+func TestParsePrefixes(t *testing.T) {
+	for _, c := range []struct {
+		in     string
+		v4, v6 int
+		ok     bool
+	}{{"24", 24, -1, true}, {"24//64", 24, 64, true}, {"/64", -1, 64, true}, {"33", 0, 0, false}, {"x", 0, 0, false}, {"//129", 0, 0, false}} {
+		v4, v6, ok := parsePrefixes(c.in)
+		if ok != c.ok || (ok && (v4 != c.v4 || v6 != c.v6)) {
+			t.Errorf("%q: got %d %d %v", c.in, v4, v6, ok)
+		}
+	}
+}
+
+func TestDMARC(t *testing.T) {
+	f := fakeDNS{txt: map[string][]string{
+		"_dmarc.reject.test":  {"v=DMARC1; p=reject; rua=mailto:r@reject.test"},
+		"_dmarc.monitor.test": {"v=DMARC1; p=none; rua=mailto:r@monitor.test"},
+		"_dmarc.norua.test":   {"v=DMARC1; p=quarantine"},
+		"_dmarc.broken.test":  {"v=DMARC1; rua=mailto:r@broken.test"},
+		"_dmarc.parent.test":  {"v=DMARC1; p=reject; rua=mailto:r@parent.test"},
+		"_dmarc.subpol.test":  {"v=DMARC1; p=reject; sp=none; rua=mailto:r@subpol.test"},
+	}}
+	b := f.bt()
+	for _, c := range []struct{ domain, status, contains string }{
+		{"reject.test", model.FindingPass, "reject"},
+		{"monitor.test", model.FindingWarn, "monitoring"},
+		{"norua.test", model.FindingWarn, "no rua"},
+		{"broken.test", model.FindingFail, "no valid p="},
+		{"none.test", model.FindingNone, "no DMARC record"},
+		{"news.parent.test", model.FindingPass, "inherited from parent.test"},
+		{"news.subpol.test", model.FindingWarn, "monitoring"}, // sp=none applies to the subdomain
+	} {
+		got := b.checkDMARC(c.domain)
+		if got.Status != c.status || !strings.Contains(got.Detail, c.contains) {
+			t.Errorf("%s: got %q %q, want %s containing %q", c.domain, got.Status, got.Detail, c.status, c.contains)
+		}
+	}
+	if h := b.checkDMARC("none.test").Hint; !strings.Contains(h, "_dmarc.none.test") || !strings.Contains(h, "rua=") {
+		t.Errorf("hint should give the record to publish: %q", h)
+	}
+}
+
+func TestMTASTSFinding(t *testing.T) {
+	f := fakeDNS{sts: map[string]*STSPolicy{
+		"enforce.test": {Mode: "enforce"}, "testing.test": {Mode: "testing"}, "off.test": {Mode: "none"},
+	}}
+	b := f.bt()
+	for domain, want := range map[string]string{
+		"enforce.test": model.FindingPass, "testing.test": model.FindingWarn, "off.test": model.FindingNone, "absent.test": model.FindingNone,
+	} {
+		if got := b.checkMTASTS(domain); got.Status != want {
+			t.Errorf("%s: %+v, want %s", domain, got, want)
+		}
+	}
+	if b.checkMTASTS("absent.test").Hint == "" {
+		t.Error("a missing policy should come with a hint")
+	}
+}
+
+func TestBlocklists(t *testing.T) {
+	f := fakeDNS{bl: map[string][]string{
+		"7.113.0.203.zen.test":    {"127.0.0.2"},
+		"7.113.0.203.refuse.test": {"127.255.255.254"}, // an open-resolver refusal is no answer
+		"7.113.0.203.also.test":   {"127.0.0.4"},
+	}}
+	f.bl["7.113.0.203.clean.test"] = nil
+	cfg := f.cfg()
+	cfg.MailBlocklists = []string{"zen.test", "clean.test", "refuse.test", "also.test"}
+	b := &Blacktip{cfg: cfg}
+	listed, answered := b.checkBlocklists("203.0.113.7")
+	if strings.Join(listed, ",") != "also.test,zen.test" {
+		t.Errorf("listed = %v", listed)
+	}
+	if answered != 3 {
+		t.Errorf("answered = %d, want 3 (the refusing zone doesn't count)", answered)
+	}
+	if l, n := b.checkBlocklists("2001:db8::1"); l != nil || n != 0 {
+		t.Errorf("IPv6 should be skipped, got %v %d", l, n)
+	}
+	cfg.MailLookupDNSBL = func(string) ([]string, error) { return nil, errors.New("timeout") }
+	if l, n := (&Blacktip{cfg: cfg}).checkBlocklists("203.0.113.7"); len(l) != 0 || n != 0 {
+		t.Errorf("errors count as no answer, got %v %d", l, n)
+	}
+}
+
+func TestCheckDomainsUsesSigningDomains(t *testing.T) {
+	f := fakeDNS{txt: map[string][]string{"example.com": {"v=spf1 ip4:203.0.113.7 -all"}, "_dmarc.example.com": {"v=DMARC1; p=reject; rua=mailto:a@example.com"}}}
+	b := f.bt()
+	pol := model.MailPolicy{ID: "m1", SourceIP: "203.0.113.7", DKIM: []model.DKIMSigner{
+		{Domain: "Example.com", Selector: "a"}, {Domain: "example.com", Selector: "b"}, {Domain: "other.test", Selector: "a"}}}
+	got := b.checkDomains(pol)
+	if len(got) != 2 || got["example.com"].SPF.Status != model.FindingPass || got["example.com"].DMARC.Status != model.FindingPass {
+		t.Fatalf("%+v", got)
+	}
+	if got["other.test"].SPF.Status != model.FindingNone {
+		t.Errorf("other.test: %+v", got["other.test"])
+	}
+	pol.Smarthost = "smtp.relay.test:587"
+	if s := b.checkDomains(pol)["example.com"].SPF; s.Status != model.FindingWarn || !strings.Contains(s.Detail, "smarthost") {
+		t.Errorf("smarthost SPF: %+v", s)
+	}
+	if b.checkDomains(model.MailPolicy{SourceIP: "203.0.113.7"}) != nil {
+		t.Error("no signing keys means no sending domains to check")
+	}
+}
+
+func TestOutcomes(t *testing.T) {
+	var o outcomeLog
+	now := time.Date(2026, 10, 27, 12, 30, 0, 0, time.UTC)
+	o.add("m1", "gmail.com", 250, "", now.Add(-26*time.Hour)) // outside the window
+	o.add("m1", "gmail.com", 250, "", now.Add(-90*time.Minute))
+	o.add("m1", "gmail.com", 250, "", now)
+	o.add("m1", "gmail.com", 451, "4.7.0 greylisted", now.Add(time.Minute))
+	o.add("m1", "yahoo.com", 550, "5.7.1 blocked", now.Add(2*time.Minute))
+	o.add("m2", "gmail.com", 250, "", now)
+
+	s := o.summary("m1", now.Add(3*time.Minute))
+	if s == nil || s.Delivered != 2 || s.Deferred != 1 || s.Failed != 1 {
+		t.Fatalf("summary: %+v", s)
+	}
+	if g := s.ByDomain["gmail.com"]; g.Delivered != 2 || g.Deferred != 1 {
+		t.Errorf("gmail: %+v", g)
+	}
+	if len(s.Recent) != 2 || s.Recent[0].Domain != "yahoo.com" || s.Recent[0].Code != 550 {
+		t.Errorf("recent should list newest first: %+v", s.Recent)
+	}
+	if o.summary("nobody", now) != nil {
+		t.Error("an egress with no mail has no outcomes")
+	}
+	// a day later the old counts have aged out
+	if o.summary("m1", now.Add(25*time.Hour)) != nil {
+		t.Error("outcomes older than the window should be gone")
+	}
+	for i := 0; i < 30; i++ {
+		o.add("m3", "x.test", 550, "no", now.Add(time.Duration(i)*time.Second))
+	}
+	if n := len(o.summary("m3", now.Add(time.Minute)).Recent); n != outcomeRecent {
+		t.Errorf("recent failures kept = %d, want %d", n, outcomeRecent)
+	}
+}
+
+func TestReportCarriesOutcomes(t *testing.T) {
+	b := &Blacktip{cfg: Config{}}
+	b.mail = map[string]model.MailPolicy{"m1": {ID: "m1"}}
+	b.mailChecks = map[string]model.MailCheck{}
+	b.recordOutcome("m1", "gmail.com", 250, "")
+	o := b.outcomes.summary("m1", time.Now())
+	if o == nil || o.Delivered != 1 {
+		t.Fatalf("%+v", o)
+	}
+}

deploy/build.sh

@@ -8,12 +8,13 @@
 #   dist/blacktip_<version>_<os>_<arch>.tar.gz or .zip
 #   dist/SHA256SUMS
 #
-# Set TARGETS="linux/amd64 linux/arm64" to build fewer platforms.
+# Releases are published for linux/amd64 and windows/amd64 only. Set TARGETS="linux/arm64 darwin/arm64"
+# (any GOOS/GOARCH pair Go supports) to build something else by hand.
 set -eu
 NAME=blacktip
 cd "$(dirname "$0")/.."
 VERSION=${1:-$(git describe --tags --always 2>/dev/null || echo dev)}
-TARGETS=${TARGETS:-"linux/amd64 linux/arm64 linux/arm windows/amd64 darwin/amd64 darwin/arm64"}
+TARGETS=${TARGETS:-"linux/amd64 windows/amd64"}
 
 rm -rf dist
 mkdir -p dist/bin dist/stage

deploy/publish-release.sh

@@ -42,19 +42,30 @@ else
 	echo "release $TAG already exists (id $id): replacing its files"
 fi
 
-existing=$(curl -fsS -H "$AUTH" "$API/releases/$id/assets" | json 'import sys, json
+# the ids of the release's assets with a given name (a failed upload can leave a partial one behind)
+asset_ids() {
+	curl -fsS -H "$AUTH" "$API/releases/$id/assets" | "$PY" -c 'import sys, json
 for a in json.load(sys.stdin):
-    print(a["id"], a["name"])')
+    if a["name"] == sys.argv[1]:
+        print(a["id"])' "$1"
+}
 
 for f in dist/*.tar.gz dist/*.zip dist/SHA256SUMS; do
 	[ -f "$f" ] || continue
 	name=$(basename "$f")
-	echo "$existing" | while read -r aid aname; do
-		if [ "$aname" = "$name" ]; then
+	n=1
+	while :; do
+		for aid in $(asset_ids "$name"); do
 			curl -fsS -X DELETE -H "$AUTH" "$API/releases/$id/assets/$aid" > /dev/null
-		fi
 		done
-	echo "uploading $name"
-	curl -fsS -X POST -H "$AUTH" -F "attachment=@$f" "$API/releases/$id/assets?name=$name" > /dev/null
+		echo "uploading $name (attempt $n)"
+		if curl -fsS --max-time 1200 -X POST -H "$AUTH" -F "attachment=@$f" "$API/releases/$id/assets?name=$name" > /dev/null; then
+			break
+		fi
+		# large uploads can hit a proxy timeout (HTTP 502); try again
+		[ "$n" -lt 5 ] || { echo "giving up on $name after $n attempts" >&2; exit 1; }
+		n=$((n + 1))
+		sleep $((n * 10))
+	done
 done
 echo "published $SERVER/$REPO/releases/tag/$TAG"

go.mod

@@ -3,7 +3,7 @@ module git.veillax.com/Veillax/blacktip
 go 1.27.3
 
 require (
-	git.veillax.com/Veillax/reef v0.1.3
+	git.veillax.com/Veillax/reef v0.1.11
 	github.com/emersion/go-msgauth v0.7.0
 	golang.org/x/crypto v0.58.0
 )

go.sum

@@ -1,7 +1,5 @@
-git.veillax.com/Veillax/reef v0.1.0 h1:O0OhEKFx2CsvACJ20sfzJNjGsngomtfDdKdxu4ROpCs=
-git.veillax.com/Veillax/reef v0.1.0/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
-git.veillax.com/Veillax/reef v0.1.3 h1:RxSq2xwaup1+MieAkhlvOTZnYwPf19thFHEAe/7VxsI=
-git.veillax.com/Veillax/reef v0.1.3/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
+git.veillax.com/Veillax/reef v0.1.11 h1:kbjgxJpmpp1HRckbW1tqJUm5/ftgvSlmUHAhFXugOlY=
+git.veillax.com/Veillax/reef v0.1.11/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
 github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g=
 github.com/Microsoft/go-winio v0.6.4/go.mod h1:1xl5JXG4ze/m74GH6Dc4Mwxfc3VL5siRVQjIAlHH07U=
 github.com/emersion/go-msgauth v0.7.0 h1:vj2hMn6KhFtW41kshIBTXvp6KgYSqpA/ZN9Pv4g1INc=

listener.go

@@ -35,12 +35,12 @@ type listener struct {
 	sessions map[string]*udpSession
 }
 
-func (b *Blacktip) open(ms []*mapState, off int, redirect bool) (*listener, error) {
+func (b *Blacktip) open(ip string, ms []*mapState, off int, redirect bool) (*listener, error) {
 	m := ms[0].m
-	addr := net.JoinHostPort(m.BindIP, strconv.Itoa(m.Port+off))
+	addr := net.JoinHostPort(ip, strconv.Itoa(m.Port+off))
 	l := &listener{b: b, kind: m.Proto, redirect: redirect, maps: ms, off: off, done: make(chan struct{}), sessions: map[string]*udpSession{}}
 	if redirect {
-		addr = net.JoinHostPort(m.BindIP, "80")
+		addr = net.JoinHostPort(ip, "80")
 		l.kind = model.ProtoHTTP
 	}
 	if m.Proto == model.ProtoUDP {
@@ -227,9 +227,7 @@ func (l *listener) handleTCP(c net.Conn) {
 	}
 	in, out := tunnel.Proxy(c, c, up, up)
 	in += int64(len(first))
-	ms.conns.Add(1)
-	ms.bytesIn.Add(in)
-	ms.bytesOut.Add(out)
+	ms.record(c.RemoteAddr().String(), 1, in, out)
 	entry.BytesIn, entry.BytesOut, entry.Millis = in, out, time.Since(start).Milliseconds()
 	l.b.logAccess(entry)
 }
@@ -363,13 +361,13 @@ func (l *listener) serveUDP() {
 			l.mu.Lock()
 			l.sessions[key] = s
 			l.mu.Unlock()
-			ms.conns.Add(1)
+			ms.record(addr.String(), 1, 0, 0)
 			go l.pumpBack(ms, key, addr, s)
 		}
 		l.mu.Lock()
 		s.last = time.Now()
 		l.mu.Unlock()
-		ms.bytesIn.Add(int64(n))
+		ms.record(addr.String(), 0, int64(n), 0)
 		s.conn.Write(buf[:n])
 	}
 }
@@ -394,7 +392,7 @@ func (l *listener) pumpBack(ms *mapState, key string, addr net.Addr, s *udpSessi
 		if _, err := l.udp.WriteTo(p, addr); err != nil {
 			return
 		}
-		ms.bytesOut.Add(int64(len(p)))
+		ms.record(addr.String(), 0, 0, int64(len(p)))
 		l.mu.Lock()
 		s.last = time.Now()
 		l.mu.Unlock()

mailegress.go

@@ -344,6 +344,7 @@ func (s *mailSession) rcpt(m tunnel.MailMsg) {
 	}
 	if err := s.cl.Rcpt(m.To); err != nil {
 		e := asSMTPErr(err)
+		s.b.recordOutcome(s.pol.ID, s.domain, e.Code, e.Text)
 		if s.rcpts == 0 {
 			s.close()
 		}
@@ -398,6 +399,7 @@ func (s *mailSession) logEntry(size, code int, result, domain, mx string) {
 	if s.note != "" {
 		path += " · " + s.note
 	}
+	s.b.recordOutcome(s.pol.ID, domain, code, strings.TrimPrefix(result, "mail-failed: "))
 	s.b.logAccess(model.LogEntry{Time: s.start, Mapping: s.pol.ID, Remote: s.peer, Method: "MAIL", Host: domain, Path: path,
 		Status: code, Result: result, BytesIn: int64(size), Millis: time.Since(s.start).Milliseconds()})
 }
@@ -429,6 +431,8 @@ func (b *Blacktip) mailCheck(pol model.MailPolicy) model.MailCheck {
 	if len(pol.DKIM) > 0 {
 		chk.DKIM = b.checkDKIM(pol.DKIM)
 	}
+	chk.Domains = b.checkDomains(pol)
+	chk.Blocklists, chk.Listable = b.checkBlocklists(pol.SourceIP)
 	probe := b.cfg.MailProbe
 	if probe == "" {
 		probe = defaultMailProbe

outcomes.go

@@ -0,0 +1,102 @@
+package blacktip
+
+import (
+	"sort"
+	"sync"
+	"time"
+
+	"git.veillax.com/Veillax/reef/model"
+)
+
+// Delivery outcomes: every mail transaction the egress handles ends delivered,
+// deferred (a 4xx the sender will retry) or failed (a 5xx). They are counted in
+// hourly buckets for the last day, so the dashboard can show a delivery rate
+// and say which recipient domains are refusing mail and why.
+
+const (
+	outcomeWindow = 24 * time.Hour
+	outcomeRecent = 10 // refusals kept for the dashboard to explain
+)
+
+type outcomeBucket struct {
+	hour     time.Time
+	byDomain map[string]model.OutcomeCount
+}
+
+type outcomeLog struct {
+	mu      sync.Mutex
+	buckets map[string][]*outcomeBucket // by mail-egress ID, oldest first
+	recent  map[string][]model.OutcomeFailure
+}
+
+// recordOutcome counts one finished transaction (or a refusal along the way).
+func (b *Blacktip) recordOutcome(egress, domain string, code int, text string) {
+	b.outcomes.add(egress, domain, code, text, time.Now())
+}
+
+func (o *outcomeLog) add(egress, domain string, code int, text string, now time.Time) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	if o.buckets == nil {
+		o.buckets, o.recent = map[string][]*outcomeBucket{}, map[string][]model.OutcomeFailure{}
+	}
+	hour := now.Truncate(time.Hour)
+	bs := o.buckets[egress]
+	if len(bs) == 0 || !bs[len(bs)-1].hour.Equal(hour) {
+		bs = append(bs, &outcomeBucket{hour: hour, byDomain: map[string]model.OutcomeCount{}})
+	}
+	cut := 0
+	for cut < len(bs) && now.Sub(bs[cut].hour) >= outcomeWindow {
+		cut++
+	}
+	bs = bs[cut:]
+	o.buckets[egress] = bs
+	c := bs[len(bs)-1].byDomain[domain]
+	switch {
+	case code >= 200 && code < 300:
+		c.Delivered++
+	case code >= 400 && code < 500:
+		c.Deferred++
+	default:
+		c.Failed++
+	}
+	bs[len(bs)-1].byDomain[domain] = c
+	if code >= 300 {
+		r := append([]model.OutcomeFailure{{Time: now, Domain: domain, Code: code, Text: text}}, o.recent[egress]...)
+		if len(r) > outcomeRecent {
+			r = r[:outcomeRecent]
+		}
+		o.recent[egress] = r
+	}
+}
+
+// summary totals the last day for one egress, or nil if it handled no mail.
+func (o *outcomeLog) summary(egress string, now time.Time) *model.MailOutcomes {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	out := &model.MailOutcomes{ByDomain: map[string]model.OutcomeCount{}}
+	for _, bk := range o.buckets[egress] {
+		if now.Sub(bk.hour) >= outcomeWindow {
+			continue
+		}
+		if out.Since.IsZero() || bk.hour.Before(out.Since) {
+			out.Since = bk.hour
+		}
+		for d, c := range bk.byDomain {
+			t := out.ByDomain[d]
+			t.Delivered, t.Deferred, t.Failed = t.Delivered+c.Delivered, t.Deferred+c.Deferred, t.Failed+c.Failed
+			out.ByDomain[d] = t
+			out.Delivered, out.Deferred, out.Failed = out.Delivered+c.Delivered, out.Deferred+c.Deferred, out.Failed+c.Failed
+		}
+	}
+	if out.Delivered+out.Deferred+out.Failed == 0 {
+		return nil
+	}
+	for _, f := range o.recent[egress] {
+		if now.Sub(f.Time) < outcomeWindow {
+			out.Recent = append(out.Recent, f)
+		}
+	}
+	sort.SliceStable(out.Recent, func(i, j int) bool { return out.Recent[i].Time.After(out.Recent[j].Time) })
+	return out
+}

redirect_test.go

@@ -0,0 +1,44 @@
+package blacktip
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"git.veillax.com/Veillax/reef/model"
+)
+
+// A redirect mapping answers from the edge itself: no proxy, relay or target is needed.
+func TestRedirectMappingAnswersWithoutATarget(t *testing.T) {
+	b := &Blacktip{logq: make(chan model.LogEntry, 8)}
+	mk := func(m model.Mapping) *listener {
+		return &listener{b: b, kind: model.ProtoHTTP, maps: []*mapState{{b: b, m: m}}}
+	}
+	cases := []struct {
+		name string
+		m    model.Mapping
+		uri  string
+		code int
+		loc  string
+	}{
+		{"temporary, no path", model.Mapping{Hostname: "old.test", RedirectTo: "https://new.test/home"}, "/a?b=1", http.StatusFound, "https://new.test/home"},
+		{"permanent keeps path and query", model.Mapping{Hostname: "old.test", RedirectTo: "https://new.test", RedirectPermanent: true, RedirectKeepPath: true}, "/a/b?c=d", http.StatusMovedPermanently, "https://new.test/a/b?c=d"},
+	}
+	for _, c := range cases {
+		r := httptest.NewRequest("GET", "http://old.test"+c.uri, nil)
+		w := httptest.NewRecorder()
+		mk(c.m).ServeHTTP(w, r)
+		if w.Code != c.code || w.Header().Get("Location") != c.loc {
+			t.Errorf("%s: got %d %q, want %d %q", c.name, w.Code, w.Header().Get("Location"), c.code, c.loc)
+		}
+	}
+	// a hostname nobody serves is still a 404
+	w := httptest.NewRecorder()
+	mk(cases[0].m).ServeHTTP(w, httptest.NewRequest("GET", "http://other.test/", nil))
+	if w.Code != http.StatusNotFound {
+		t.Errorf("unknown host: got %d", w.Code)
+	}
+	if e := <-b.logq; e.Result != "redirect" || e.Status != http.StatusFound {
+		t.Errorf("access log entry: %+v", e)
+	}
+}

traefik.go

@@ -49,6 +49,7 @@ type traefikMgr struct {
 	stop      context.CancelFunc // stops the current traefik process
 	lastErr   string
 	maps      map[string]*mapState
+	binds     map[string][]string // mapping ID -> the addresses it listens on (a wildcard steps aside for named ones)
 	ports     map[string]int      // mapping ID -> relay port, kept stable across edits so Traefik's backend URL never changes
 	dynBytes  []byte
 }
@@ -217,20 +218,23 @@ func hostRule(host string) (string, bool) {
 
 // sync rewrites Traefik's configuration for the given web mappings and
 // (re)starts the process if its static part changed. Called with b.mu held.
-func (t *traefikMgr) sync(ctx context.Context, maps []*mapState, acmeEmail string, staging bool) {
+func (t *traefikMgr) sync(ctx context.Context, maps []*mapState, binds map[string][]string, acmeEmail string, staging bool) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
+	t.binds = binds
 	t.maps = map[string]*mapState{}
 	eps := map[string]string{} // name -> address
 	needACME := false
 	for _, ms := range maps {
 		t.maps[ms.m.ID] = ms
-		eps[epName(ms.m.BindIP, ms.m.Port)] = net.JoinHostPort(ms.m.BindIP, strconv.Itoa(ms.m.Port))
+		for _, ip := range t.bindsOf(ms.m) {
+			eps[epName(ip, ms.m.Port)] = net.JoinHostPort(ip, strconv.Itoa(ms.m.Port))
+			if ms.m.Proto == model.ProtoHTTPS && ms.m.RedirectHTTP {
+				eps[epName(ip, 80)] = net.JoinHostPort(ip, "80")
+			}
+		}
 		if ms.m.Proto == model.ProtoHTTPS {
 			needACME = true
-			if ms.m.RedirectHTTP {
-				eps[epName(ms.m.BindIP, 80)] = net.JoinHostPort(ms.m.BindIP, "80")
-			}
 		}
 	}
 	routers := map[string]any{}
@@ -240,11 +244,45 @@ func (t *traefikMgr) sync(ctx context.Context, maps []*mapState, acmeEmail strin
 	}
 	for _, ms := range maps {
 		m := ms.m
+		if m.IsRedirect() { // answered by Traefik itself: no relay, login or target
+			rule, wildcard := hostRule(m.Hostname)
+			var eps []string
+			for _, ip := range t.bindsOf(m) {
+				eps = append(eps, epName(ip, m.Port))
+			}
+			prio := 5000
+			if wildcard {
+				prio = 2000
+			}
+			to := strings.ReplaceAll(m.RedirectTo, "$", "$$") // "$" would otherwise start a group reference
+			regex, repl := "^.*", to
+			if m.RedirectKeepPath {
+				regex, repl = "^https?://[^/]+(.*)", strings.TrimRight(to, "/")+"${1}"
+			}
+			mws["rd-"+m.ID] = map[string]any{"redirectRegex": map[string]any{"regex": regex, "replacement": repl, "permanent": m.RedirectPermanent}}
+			router := map[string]any{"rule": rule, "entryPoints": eps, "service": "noop@internal", "middlewares": []string{"rd-" + m.ID}, "priority": prio}
+			if m.Proto == model.ProtoHTTPS {
+				tlsCfg := map[string]any{}
+				if !t.cfg.NoACME {
+					tlsCfg["certResolver"] = "le"
+					if wildcard {
+						tlsCfg["domains"] = []map[string]any{{"main": m.Hostname}}
+					}
+				}
+				router["tls"] = tlsCfg
+			}
+			routers["m-"+m.ID] = router
+			continue
+		}
 		if ms.relayPort == 0 {
 			continue
 		}
 		rule, wildcard := hostRule(m.Hostname)
-		ep := epName(m.BindIP, m.Port)
+		var mainEPs, redirEPs []string
+		for _, ip := range t.bindsOf(m) {
+			mainEPs = append(mainEPs, epName(ip, m.Port))
+			redirEPs = append(redirEPs, epName(ip, 80))
+		}
 		prio := 5000
 		if wildcard {
 			prio = 2000
@@ -274,7 +312,7 @@ func (t *traefikMgr) sync(ctx context.Context, maps []*mapState, acmeEmail strin
 			mws["host-"+m.ID] = map[string]any{"headers": map[string]any{"customRequestHeaders": map[string]string{"Host": m.RewriteHost}}}
 			chain = append(chain, "host-"+m.ID)
 		}
-		router := map[string]any{"rule": rule, "entryPoints": []string{ep}, "service": "s-" + m.ID, "middlewares": chain, "priority": prio}
+		router := map[string]any{"rule": rule, "entryPoints": mainEPs, "service": "s-" + m.ID, "middlewares": chain, "priority": prio}
 		var tlsCfg map[string]any
 		if m.Proto == model.ProtoHTTPS {
 			tlsCfg = map[string]any{}
@@ -292,7 +330,7 @@ func (t *traefikMgr) sync(ctx context.Context, maps []*mapState, acmeEmail strin
 			services["g-"+m.ID] = map[string]any{"loadBalancer": map[string]any{
 				"servers": []map[string]string{{"url": fmt.Sprintf("http://127.0.0.1:%d", t.gatePort)}}, "passHostHeader": true}}
 			mws["gate-"+m.ID] = map[string]any{"headers": map[string]any{"customRequestHeaders": map[string]string{"X-Hammerhead-Mapping": m.ID}}}
-			g := map[string]any{"rule": rule + " && PathPrefix(`/.hammerhead/`)", "entryPoints": []string{ep}, "service": "g-" + m.ID,
+			g := map[string]any{"rule": rule + " && PathPrefix(`/.hammerhead/`)", "entryPoints": mainEPs, "service": "g-" + m.ID,
 				"middlewares": []string{"gate-" + m.ID}, "priority": prio + 1000}
 			if tlsCfg != nil {
 				g["tls"] = tlsCfg
@@ -305,7 +343,7 @@ func (t *traefikMgr) sync(ctx context.Context, maps []*mapState, acmeEmail strin
 				redirPort = strconv.Itoa(m.Port)
 			}
 			mws["redir-"+m.ID] = map[string]any{"redirectScheme": map[string]any{"scheme": "https", "port": redirPort, "permanent": true}}
-			routers["r-"+m.ID] = map[string]any{"rule": rule, "entryPoints": []string{epName(m.BindIP, 80)}, "service": "noop@internal",
+			routers["r-"+m.ID] = map[string]any{"rule": rule, "entryPoints": redirEPs, "service": "noop@internal",
 				"middlewares": []string{"redir-" + m.ID}, "priority": prio}
 		}
 	}
@@ -506,9 +544,7 @@ func (t *traefikMgr) tailAccessLog(ctx context.Context) {
 			if ms == nil {
 				continue
 			}
-			ms.conns.Add(1)
-			ms.bytesIn.Add(tl.RequestSize)
-			ms.bytesOut.Add(tl.DownstreamSize)
+			ms.record(tl.ClientHost, 1, tl.RequestSize, tl.DownstreamSize)
 			t.b.logAccess(model.LogEntry{Time: tl.StartUTC, Mapping: id, Remote: tl.ClientHost, Method: tl.RequestMethod, Host: hostOnly(tl.RequestHost),
 				Path: tl.RequestPath, Status: tl.DownstreamStatus, Result: "allowed", BytesIn: tl.RequestSize, BytesOut: tl.DownstreamSize, Millis: tl.DurationNanos / 1e6})
 		}
@@ -545,10 +581,7 @@ func (t *traefikMgr) probeCerts(ctx context.Context) {
 		t.mu.Unlock()
 		changed := false
 		for _, ms := range targets {
-			ip := ms.m.BindIP
-			if pip := net.ParseIP(ip); pip != nil && pip.IsUnspecified() {
-				ip = "127.0.0.1"
-			}
+			ip := t.probeAddr(ms.m)
 			status := t.probeOne(net.JoinHostPort(ip, strconv.Itoa(ms.m.Port)), ms.m.Hostname)
 			t.b.mu.Lock()
 			if t.b.acme.certs == nil {
@@ -598,3 +631,27 @@ func containsStr(list []string, s string) bool {
 	}
 	return false
 }
+
+// bindsOf is the addresses a web mapping listens on; t.mu is held, or the caller does not race sync.
+func (t *traefikMgr) bindsOf(m model.Mapping) []string {
+	if ips := t.binds[m.ID]; len(ips) > 0 {
+		return ips
+	}
+	return []string{m.BindIP}
+}
+
+// probeAddr picks an address on this machine to test a mapping's certificate through.
+func (t *traefikMgr) probeAddr(m model.Mapping) string {
+	t.mu.Lock()
+	ips := t.bindsOf(m)
+	t.mu.Unlock()
+	for _, ip := range ips {
+		if ip == "127.0.0.1" {
+			return ip
+		}
+	}
+	if p := net.ParseIP(ips[0]); p != nil && p.IsUnspecified() {
+		return "127.0.0.1"
+	}
+	return ips[0]
+}

traffic.go

@@ -0,0 +1,89 @@
+package blacktip
+
+import (
+	"net"
+	"net/netip"
+	"sync"
+
+	"git.veillax.com/Veillax/reef/model"
+)
+
+// maxTrackedSources bounds the per-port table of client addresses between two reports, so a flood
+// from spoofed or scanning addresses can't grow it without limit. Addresses past it still count
+// in the port's totals; they just aren't named.
+const maxTrackedSources = 5000
+
+// sourceTable counts a port's traffic per client address until the next report takes it.
+type sourceTable struct {
+	mu sync.Mutex
+	by map[string]*model.SourceStat
+}
+
+func (t *sourceTable) add(ip string, conns, in, out int64) {
+	if ip == "" {
+		return
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	s := t.by[ip]
+	if s == nil {
+		if len(t.by) >= maxTrackedSources {
+			return
+		}
+		if t.by == nil {
+			t.by = map[string]*model.SourceStat{}
+		}
+		s = &model.SourceStat{IP: ip}
+		t.by[ip] = s
+	}
+	s.Conns += conns
+	s.BytesIn += in
+	s.BytesOut += out
+}
+
+func (t *sourceTable) take() map[string]*model.SourceStat {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	by := t.by
+	t.by = nil
+	return by
+}
+
+// clientIP reduces a "host:port" remote address to the client's IP (IPv4-mapped IPv6 as plain IPv4).
+// Anything that isn't an address gives "".
+func clientIP(remote string) string {
+	host, _, err := net.SplitHostPort(remote)
+	if err != nil {
+		host = remote
+	}
+	a, err := netip.ParseAddr(host)
+	if err != nil {
+		return ""
+	}
+	return a.WithZone("").Unmap().String()
+}
+
+// record counts traffic on a port, and names the client it came from when that is known.
+func (ms *mapState) record(remote string, conns, in, out int64) {
+	ms.conns.Add(conns)
+	ms.bytesIn.Add(in)
+	ms.bytesOut.Add(out)
+	ms.sources.add(clientIP(remote), conns, in, out)
+}
+
+// takeStats hands over (and resets) the traffic counted since the last call.
+func (ms *mapState) takeStats() (model.Stat, bool) {
+	st := model.Stat{Conns: ms.conns.Swap(0), BytesIn: ms.bytesIn.Swap(0), BytesOut: ms.bytesOut.Swap(0)}
+	st.Sources = model.TopSources(ms.sources.take(), model.MaxSources)
+	return st, st.Conns|st.BytesIn|st.BytesOut != 0
+}
+
+// restoreStats puts taken traffic back after a report that didn't arrive, so none of it is lost.
+func (ms *mapState) restoreStats(st model.Stat) {
+	ms.conns.Add(st.Conns)
+	ms.bytesIn.Add(st.BytesIn)
+	ms.bytesOut.Add(st.BytesOut)
+	for _, s := range st.Sources {
+		ms.sources.add(s.IP, s.Conns, s.BytesIn, s.BytesOut)
+	}
+}

traffic_test.go

@@ -0,0 +1,55 @@
+package blacktip
+
+import "testing"
+
+func TestClientIP(t *testing.T) {
+	for in, want := range map[string]string{
+		"203.0.113.9:5000":     "203.0.113.9",
+		"[2001:db8::1]:443":    "2001:db8::1",
+		"[::ffff:192.0.2.7]:1": "192.0.2.7",
+		"198.51.100.2":         "198.51.100.2", // no port, as Traefik reports it
+		"[fe80::1%eth0]:80":    "fe80::1",
+		"not an address":       "",
+		"":                     "",
+	} {
+		if got := clientIP(in); got != want {
+			t.Errorf("clientIP(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestStatsTakeAndRestore(t *testing.T) {
+	ms := &mapState{}
+	ms.record("203.0.113.1:1000", 1, 100, 400)
+	ms.record("203.0.113.1:1001", 1, 10, 40)
+	ms.record("203.0.113.2:2000", 1, 5, 5)
+	ms.record("garbage", 1, 1, 1) // counted in the totals but not named
+
+	st, any := ms.takeStats()
+	if !any || st.Conns != 4 || st.BytesIn != 116 || st.BytesOut != 446 {
+		t.Fatalf("totals: %+v", st)
+	}
+	if len(st.Sources) != 2 || st.Sources[0].IP != "203.0.113.1" || st.Sources[0].Conns != 2 || st.Sources[0].BytesOut != 440 {
+		t.Fatalf("sources: %+v", st.Sources)
+	}
+	if _, any := ms.takeStats(); any {
+		t.Fatal("counters were not reset")
+	}
+
+	ms.record("203.0.113.1:1", 1, 1, 1) // traffic that arrived before the failed report was retried
+	ms.restoreStats(st)
+	again, _ := ms.takeStats()
+	if again.Conns != 5 || again.BytesIn != 117 || again.Sources[0].Conns != 3 {
+		t.Fatalf("after restore: %+v", again)
+	}
+}
+
+func TestSourceTableIsBounded(t *testing.T) {
+	var tb sourceTable
+	for i := 0; i < maxTrackedSources+50; i++ {
+		tb.add(string(rune('a'+i%26))+"."+string(rune('0'+i/26%10))+"."+string(rune(i)), 1, 1, 1)
+	}
+	if n := len(tb.take()); n > maxTrackedSources {
+		t.Fatalf("tracked %d addresses", n)
+	}
+}

web.go

@@ -258,12 +258,20 @@ func (l *listener) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 		entry.BytesOut = rec.bytes
 		entry.Millis = time.Since(start).Milliseconds()
-		ms.conns.Add(1)
-		ms.bytesIn.Add(entry.BytesIn)
-		ms.bytesOut.Add(entry.BytesOut)
+		ms.record(r.RemoteAddr, 1, entry.BytesIn, entry.BytesOut)
 		b.logAccess(entry)
 	}()
 
+	if ms.m.IsRedirect() {
+		code := http.StatusFound
+		if ms.m.RedirectPermanent {
+			code = http.StatusMovedPermanently
+		}
+		entry.Result = "redirect"
+		http.Redirect(rec, r, ms.m.RedirectLocation(r.URL.RequestURI()), code)
+		return
+	}
+
 	secure := l.kind == model.ProtoHTTPS
 	scheme := "http"
 	if secure {

yield_test.go

@@ -0,0 +1,37 @@
+package blacktip
+
+import (
+	"testing"
+
+	"git.veillax.com/Veillax/reef/model"
+)
+
+func TestWildcardWebsitesStepAsideForNamedAddresses(t *testing.T) {
+	all := []model.Mapping{
+		{ID: "a", Proto: model.ProtoHTTPS, BindIP: "0.0.0.0", Port: 443, Hostname: "a.test"},
+		{ID: "b", Proto: model.ProtoHTTPS, BindIP: "127.0.0.1", Port: 443, Hostname: "b.test"},
+		{ID: "c", Proto: model.ProtoHTTPS, BindIP: "0.0.0.0", Port: 8443, Hostname: "c.test"},
+		{ID: "d", Proto: model.ProtoTCP, BindIP: "0.0.0.0", Port: 443},
+	}
+	// the wildcard on 443 gives up the address that "b" names
+	for _, ip := range bindIPs(all[0], all) {
+		if ip == "127.0.0.1" {
+			t.Fatalf("wildcard still listens on 127.0.0.1: %v", bindIPs(all[0], all))
+		}
+	}
+	// the named address keeps its own listener
+	if got := bindIPs(all[1], all); len(got) != 1 || got[0] != "127.0.0.1" {
+		t.Fatalf("named mapping: %v", got)
+	}
+	// a different port, or a raw TCP port, is not affected
+	if got := bindIPs(all[2], all); len(got) != 1 || got[0] != "0.0.0.0" {
+		t.Fatalf("other port: %v", got)
+	}
+	if got := bindIPs(all[3], all); len(got) != 1 || got[0] != "0.0.0.0" {
+		t.Fatalf("raw tcp: %v", got)
+	}
+	// with nothing named on that port the wildcard stays a single wildcard listener
+	if got := bindIPs(all[0], all[:1]); len(got) != 1 || got[0] != "0.0.0.0" {
+		t.Fatalf("alone: %v", got)
+	}
+}