model: terminal sessions in the site config, and the agent's SSH server in its report

SiteConfig.Terminals lists terminal sessions waiting for the agent, which calls
back to Hammerhead and joins the connection to its SSH server. SiteReport and
Site carry the SSH server's address and whether the agent can relay.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

README.md

@@ -1,6 +1,6 @@
 # Reef
 
-A shared library for Hammerhead, Blacktip, and Bamboo
+A shared library for Hammerhead, Blacktip, Bamboo and Sandbar
 
 Shared types and building blocks for the sharks stack: [Hammerhead](https://git.veillax.com/Veillax/hammerhead) (control plane), [Blacktip](https://git.veillax.com/Veillax/blacktip) (edge) and [Bamboo](https://git.veillax.com/Veillax/bamboo) (site agent).
 
@@ -15,6 +15,7 @@ Reef holds the code all three programs agree on, so each can be built and releas
 | `netguard` | Refuses internal addresses for anything that connects out on a user's behalf (webhooks, mail delivery), and dials the address it checked |
 | `docker` | Lists running containers through the Docker Engine API, with no SDK |
 | `dotenv` | Loads a `.env` file into the environment without overriding real variables |
+| `secretbox` | AES-256-GCM sealing of small secrets (`enc1:` values) before they are written to a database |
 
 ## Using it
 

dmarc/dmarc.go

@@ -0,0 +1,274 @@
+// Package dmarc parses DMARC aggregate (rua) reports (RFC 7489 appendix A).
+// Receivers send them as XML, usually gzip or zip compressed, usually as an
+// email attachment; Parse accepts any of those forms.
+package dmarc
+
+import (
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"encoding/xml"
+	"errors"
+	"io"
+	"mime"
+	"mime/multipart"
+	"mime/quotedprintable"
+	"net/mail"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// maxXML caps a decompressed report; aggregate reports are small, so a larger one is a zip bomb.
+const maxXML = 32 << 20
+
+// Report is one receiver's summary of the mail it saw claiming to be from Domain.
+type Report struct {
+	Org      string    `json:"org"` // the receiver that sent the report
+	Email    string    `json:"email"`
+	ReportID string    `json:"reportId"`
+	Begin    time.Time `json:"begin"`
+	End      time.Time `json:"end"`
+	Domain   string    `json:"domain"`
+	Policy   Policy    `json:"policy"`
+	Records  []Record  `json:"records"`
+}
+
+// Policy is the DMARC policy the receiver applied.
+type Policy struct {
+	P     string `json:"p"`   // none, quarantine or reject
+	SP    string `json:"sp"`  // for subdomains
+	Pct   int    `json:"pct"` // percent of mail the policy applies to
+	ADKIM string `json:"adkim"`
+	ASPF  string `json:"aspf"`
+}
+
+// Record is one row: a sending IP and what happened to its mail.
+type Record struct {
+	SourceIP     string       `json:"sourceIp"`
+	Count        int          `json:"count"`
+	Disposition  string       `json:"disposition"` // what the receiver did: none, quarantine, reject
+	DKIM         string       `json:"dkim"`        // aligned DKIM result: pass or fail
+	SPF          string       `json:"spf"`         // aligned SPF result: pass or fail
+	HeaderFrom   string       `json:"headerFrom"`
+	EnvelopeFrom string       `json:"envelopeFrom,omitempty"`
+	DKIMAuth     []AuthResult `json:"dkimAuth,omitempty"`
+	SPFAuth      []AuthResult `json:"spfAuth,omitempty"`
+}
+
+// AuthResult is a raw DKIM or SPF verdict, before alignment with the From domain.
+type AuthResult struct {
+	Domain   string `json:"domain"`
+	Selector string `json:"selector,omitempty"`
+	Result   string `json:"result"`
+}
+
+// Passed reports whether the row's mail passed DMARC: either aligned check passing is enough.
+func (r Record) Passed() bool { return r.DKIM == "pass" || r.SPF == "pass" }
+
+type xmlFeedback struct {
+	Meta struct {
+		Org      string `xml:"org_name"`
+		Email    string `xml:"email"`
+		ReportID string `xml:"report_id"`
+		Range    struct {
+			Begin int64 `xml:"begin"`
+			End   int64 `xml:"end"`
+		} `xml:"date_range"`
+	} `xml:"report_metadata"`
+	Policy struct {
+		Domain string `xml:"domain"`
+		ADKIM  string `xml:"adkim"`
+		ASPF   string `xml:"aspf"`
+		P      string `xml:"p"`
+		SP     string `xml:"sp"`
+		Pct    string `xml:"pct"`
+	} `xml:"policy_published"`
+	Records []struct {
+		Row struct {
+			SourceIP string `xml:"source_ip"`
+			Count    int    `xml:"count"`
+			Eval     struct {
+				Disposition string `xml:"disposition"`
+				DKIM        string `xml:"dkim"`
+				SPF         string `xml:"spf"`
+			} `xml:"policy_evaluated"`
+		} `xml:"row"`
+		Identifiers struct {
+			HeaderFrom   string `xml:"header_from"`
+			EnvelopeFrom string `xml:"envelope_from"`
+		} `xml:"identifiers"`
+		Auth struct {
+			DKIM []struct {
+				Domain   string `xml:"domain"`
+				Selector string `xml:"selector"`
+				Result   string `xml:"result"`
+			} `xml:"dkim"`
+			SPF []struct {
+				Domain string `xml:"domain"`
+				Result string `xml:"result"`
+			} `xml:"spf"`
+		} `xml:"auth_results"`
+	} `xml:"record"`
+}
+
+// Parse reads a report from raw XML, gzip, zip, or a whole email carrying one of those.
+func Parse(data []byte) (*Report, error) {
+	xmlData, err := unwrap(data, 0)
+	if err != nil {
+		return nil, err
+	}
+	var f xmlFeedback
+	if err := xml.Unmarshal(xmlData, &f); err != nil {
+		return nil, errors.New("not a DMARC report: " + err.Error())
+	}
+	if f.Policy.Domain == "" {
+		return nil, errors.New("not a DMARC report: no published policy domain")
+	}
+	if strings.TrimSpace(f.Meta.ReportID) == "" {
+		return nil, errors.New("not a DMARC report: no report_id")
+	}
+	pct := 100
+	if n, err := strconv.Atoi(strings.TrimSpace(f.Policy.Pct)); err == nil && n >= 0 && n <= 100 {
+		pct = n
+	}
+	r := &Report{
+		Org: strings.TrimSpace(f.Meta.Org), Email: strings.TrimSpace(f.Meta.Email), ReportID: strings.TrimSpace(f.Meta.ReportID),
+		Begin:  time.Unix(f.Meta.Range.Begin, 0).UTC(),
+		End:    time.Unix(f.Meta.Range.End, 0).UTC(),
+		Domain: lower(f.Policy.Domain),
+		Policy: Policy{P: lower(f.Policy.P), SP: lower(f.Policy.SP), Pct: pct, ADKIM: lower(f.Policy.ADKIM), ASPF: lower(f.Policy.ASPF)},
+	}
+	for _, x := range f.Records {
+		rec := Record{
+			SourceIP: strings.TrimSpace(x.Row.SourceIP), Count: x.Row.Count,
+			Disposition: lower(x.Row.Eval.Disposition), DKIM: lower(x.Row.Eval.DKIM), SPF: lower(x.Row.Eval.SPF),
+			HeaderFrom: lower(x.Identifiers.HeaderFrom), EnvelopeFrom: lower(x.Identifiers.EnvelopeFrom),
+		}
+		for _, d := range x.Auth.DKIM {
+			rec.DKIMAuth = append(rec.DKIMAuth, AuthResult{Domain: lower(d.Domain), Selector: d.Selector, Result: lower(d.Result)})
+		}
+		for _, s := range x.Auth.SPF {
+			rec.SPFAuth = append(rec.SPFAuth, AuthResult{Domain: lower(s.Domain), Result: lower(s.Result)})
+		}
+		r.Records = append(r.Records, rec)
+	}
+	return r, nil
+}
+
+func lower(s string) string { return strings.ToLower(strings.TrimSpace(s)) }
+
+// unwrap peels gzip, zip and email layers until XML is left.
+func unwrap(data []byte, depth int) ([]byte, error) {
+	if depth > 4 {
+		return nil, errors.New("report is wrapped too deeply")
+	}
+	switch {
+	case bytes.HasPrefix(data, []byte{0x1f, 0x8b}):
+		zr, err := gzip.NewReader(bytes.NewReader(data))
+		if err != nil {
+			return nil, err
+		}
+		defer zr.Close()
+		return readLimited(zr, depth)
+	case bytes.HasPrefix(data, []byte("PK\x03\x04")):
+		zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+		if err != nil {
+			return nil, err
+		}
+		for _, f := range zr.File {
+			if f.FileInfo().IsDir() {
+				continue
+			}
+			rc, err := f.Open()
+			if err != nil {
+				return nil, err
+			}
+			out, err := readLimited(rc, depth)
+			rc.Close()
+			return out, err
+		}
+		return nil, errors.New("empty zip")
+	}
+	trimmed := bytes.TrimSpace(data)
+	if bytes.HasPrefix(trimmed, []byte("<")) {
+		return trimmed, nil
+	}
+	return fromEmail(data, depth)
+}
+
+func readLimited(r io.Reader, depth int) ([]byte, error) {
+	b, err := io.ReadAll(io.LimitReader(r, maxXML+1))
+	if err != nil {
+		return nil, err
+	}
+	if len(b) > maxXML {
+		return nil, errors.New("report is too large")
+	}
+	return unwrap(b, depth+1)
+}
+
+// fromEmail finds the report attachment in a message.
+func fromEmail(data []byte, depth int) ([]byte, error) {
+	m, err := mail.ReadMessage(bytes.NewReader(data))
+	if err != nil {
+		return nil, errors.New("not a DMARC report: unrecognised format")
+	}
+	b, err := fromPart(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body, depth)
+	if err != nil {
+		return nil, err
+	}
+	if b == nil {
+		return nil, errors.New("no report attached to the email")
+	}
+	return b, nil
+}
+
+func fromPart(contentType, encoding string, body io.Reader, depth int) ([]byte, error) {
+	mt, params, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		mt = "text/plain"
+	}
+	if strings.HasPrefix(mt, "multipart/") {
+		mr := multipart.NewReader(body, params["boundary"])
+		for {
+			p, err := mr.NextPart()
+			if err != nil {
+				return nil, nil
+			}
+			b, err := fromPart(p.Header.Get("Content-Type"), p.Header.Get("Content-Transfer-Encoding"), p, depth)
+			if err != nil || b != nil {
+				return b, err
+			}
+		}
+	}
+	switch mt {
+	case "application/gzip", "application/x-gzip", "application/zip", "application/x-zip-compressed", "application/xml", "text/xml", "application/octet-stream":
+		raw, err := io.ReadAll(io.LimitReader(decodeBody(encoding, body), maxXML+1))
+		if err != nil {
+			return nil, err
+		}
+		return unwrap(raw, depth+1)
+	}
+	return nil, nil
+}
+
+func decodeBody(encoding string, r io.Reader) io.Reader {
+	switch strings.ToLower(strings.TrimSpace(encoding)) {
+	case "base64":
+		// mail wraps base64 in lines, which the decoder ignores when it is given the line breaks stripped
+		b, _ := io.ReadAll(io.LimitReader(r, maxXML*2))
+		clean := strings.Map(func(c rune) rune {
+			if c == '\r' || c == '\n' || c == ' ' || c == '\t' {
+				return -1
+			}
+			return c
+		}, string(b))
+		out, _ := base64.StdEncoding.DecodeString(clean)
+		return bytes.NewReader(out)
+	case "quoted-printable":
+		return quotedprintable.NewReader(r)
+	}
+	return r
+}

dmarc/dmarc_test.go

@@ -0,0 +1,157 @@
+package dmarc
+
+import (
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"strings"
+	"testing"
+)
+
+const sample = `<?xml version="1.0" encoding="UTF-8" ?>
+<feedback>
+  <report_metadata>
+    <org_name>google.com</org_name>
+    <email>noreply-dmarc-support@google.com</email>
+    <report_id>12345</report_id>
+    <date_range><begin>1700000000</begin><end>1700086399</end></date_range>
+  </report_metadata>
+  <policy_published>
+    <domain>Example.COM</domain>
+    <adkim>r</adkim><aspf>r</aspf><p>none</p><sp>none</sp><pct>100</pct>
+  </policy_published>
+  <record>
+    <row>
+      <source_ip>203.0.113.7</source_ip><count>12</count>
+      <policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>fail</spf></policy_evaluated>
+    </row>
+    <identifiers><header_from>example.com</header_from><envelope_from>bounce.example.com</envelope_from></identifiers>
+    <auth_results>
+      <dkim><domain>example.com</domain><selector>s1</selector><result>pass</result></dkim>
+      <spf><domain>bounce.example.com</domain><result>pass</result></spf>
+    </auth_results>
+  </record>
+  <record>
+    <row>
+      <source_ip>198.51.100.9</source_ip><count>3</count>
+      <policy_evaluated><disposition>quarantine</disposition><dkim>fail</dkim><spf>fail</spf></policy_evaluated>
+    </row>
+    <identifiers><header_from>example.com</header_from></identifiers>
+    <auth_results>
+      <spf><domain>example.com</domain><result>softfail</result></spf>
+    </auth_results>
+  </record>
+</feedback>`
+
+func check(t *testing.T, r *Report) {
+	t.Helper()
+	if r.Org != "google.com" || r.ReportID != "12345" || r.Domain != "example.com" {
+		t.Fatalf("metadata: %+v", r)
+	}
+	if r.Begin.Unix() != 1700000000 || r.End.Unix() != 1700086399 {
+		t.Fatalf("range: %v %v", r.Begin, r.End)
+	}
+	if r.Policy.P != "none" || r.Policy.Pct != 100 || r.Policy.ADKIM != "r" {
+		t.Fatalf("policy: %+v", r.Policy)
+	}
+	if len(r.Records) != 2 {
+		t.Fatalf("records: %d", len(r.Records))
+	}
+	a, b := r.Records[0], r.Records[1]
+	if a.SourceIP != "203.0.113.7" || a.Count != 12 || !a.Passed() || a.EnvelopeFrom != "bounce.example.com" {
+		t.Fatalf("first record: %+v", a)
+	}
+	if len(a.DKIMAuth) != 1 || a.DKIMAuth[0].Selector != "s1" || len(a.SPFAuth) != 1 {
+		t.Fatalf("auth results: %+v", a)
+	}
+	if b.Passed() || b.Disposition != "quarantine" || b.SPFAuth[0].Result != "softfail" {
+		t.Fatalf("second record: %+v", b)
+	}
+}
+
+func gz(t *testing.T, s string) []byte {
+	var buf bytes.Buffer
+	w := gzip.NewWriter(&buf)
+	w.Write([]byte(s))
+	w.Close()
+	return buf.Bytes()
+}
+
+func zipped(t *testing.T, s string) []byte {
+	var buf bytes.Buffer
+	w := zip.NewWriter(&buf)
+	f, _ := w.Create("report.xml")
+	f.Write([]byte(s))
+	w.Close()
+	return buf.Bytes()
+}
+
+func email(contentType, encoding string, body []byte) []byte {
+	b64 := base64.StdEncoding.EncodeToString(body)
+	var wrapped strings.Builder
+	for len(b64) > 76 {
+		wrapped.WriteString(b64[:76] + "\r\n")
+		b64 = b64[76:]
+	}
+	wrapped.WriteString(b64 + "\r\n")
+	return []byte("From: noreply@google.com\r\nSubject: Report Domain: example.com\r\nMIME-Version: 1.0\r\n" +
+		"Content-Type: multipart/mixed; boundary=\"XX\"\r\n\r\n--XX\r\nContent-Type: text/plain\r\n\r\nSee attached.\r\n" +
+		"--XX\r\nContent-Type: " + contentType + "\r\nContent-Transfer-Encoding: " + encoding + "\r\n\r\n" + wrapped.String() + "--XX--\r\n")
+}
+
+func TestParseForms(t *testing.T) {
+	forms := map[string][]byte{
+		"xml":        []byte("\n" + sample),
+		"gzip":       gz(t, sample),
+		"zip":        zipped(t, sample),
+		"mail+gzip":  email("application/gzip", "base64", gz(t, sample)),
+		"mail+zip":   email("application/zip; name=r.zip", "base64", zipped(t, sample)),
+		"mail+xml":   email("text/xml", "base64", []byte(sample)),
+		"mail+octet": email("application/octet-stream", "base64", gz(t, sample)),
+	}
+	for name, data := range forms {
+		t.Run(name, func(t *testing.T) {
+			r, err := Parse(data)
+			if err != nil {
+				t.Fatal(err)
+			}
+			check(t, r)
+		})
+	}
+}
+
+func TestParseRejects(t *testing.T) {
+	for name, data := range map[string][]byte{
+		"empty":       nil,
+		"text":        []byte("hello, this is not a report"),
+		"other xml":   []byte("<html><body/></html>"),
+		"no id":       []byte(strings.Replace(sample, "<report_id>12345</report_id>", "", 1)),
+		"mail no att": []byte("Subject: x\r\n\r\njust text\r\n"),
+		"bad gzip":    {0x1f, 0x8b, 0, 0},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if r, err := Parse(data); err == nil {
+				t.Fatalf("expected an error, got %+v", r)
+			}
+		})
+	}
+}
+
+func TestParseBombIsRefused(t *testing.T) {
+	big := "<feedback>" + strings.Repeat("x", maxXML+10) + "</feedback>"
+	if _, err := Parse(gz(t, big)); err == nil || !strings.Contains(err.Error(), "too large") {
+		t.Fatalf("want too large, got %v", err)
+	}
+}
+
+func TestPctDefaultsAndClamps(t *testing.T) {
+	r, err := Parse([]byte(strings.Replace(sample, "<pct>100</pct>", "<pct>25</pct>", 1)))
+	if err != nil || r.Policy.Pct != 25 {
+		t.Fatalf("pct 25: %v %+v", err, r)
+	}
+	r, err = Parse([]byte(strings.Replace(sample, "<pct>100</pct>", "", 1)))
+	if err != nil || r.Policy.Pct != 100 {
+		t.Fatalf("missing pct: %v %+v", err, r)
+	}
+}

model/model.go

@@ -1,7 +1,11 @@
 // Package model holds the types shared by hammerhead, blacktip and bamboo.
 package model
 
-import "time"
+import (
+	"sort"
+	"strings"
+	"time"
+)
 
 const (
 	ProtoTCP   = "tcp"   // raw TCP on ip:port
@@ -19,6 +23,21 @@ func (m Mapping) LastPort() int {
 	return m.Port
 }
 
+// IsRedirect reports whether the mapping answers with a redirect instead of forwarding to a target.
+func (m Mapping) IsRedirect() bool { return m.RedirectTo != "" }
+
+// RedirectLocation is where a request for requestURI (path and query) is sent. With KeepPath the
+// request's path and query follow the destination, so example.com/a?b=1 goes to to/a?b=1.
+func (m Mapping) RedirectLocation(requestURI string) string {
+	if !m.RedirectKeepPath {
+		return m.RedirectTo
+	}
+	if requestURI == "" || requestURI[0] != '/' {
+		requestURI = "/" + requestURI
+	}
+	return strings.TrimRight(m.RedirectTo, "/") + requestURI
+}
+
 // HostBased reports whether mappings of this protocol are routed by hostname.
 func HostBased(proto string) bool {
 	return proto == ProtoHTTP || proto == ProtoHTTPS || proto == ProtoTLS
@@ -39,6 +58,13 @@ type Edge struct {
 	OrgIDs     []string  `json:"orgIds"`            // orgs allowed to use this edge; empty = all
 	IPs        []string  `json:"ips"`               // addresses the edge reported it can bind
 	LastSeen   time.Time `json:"lastSeen"`
+
+	// Version is the release the program reports (e.g. v0.1.5). AutoUpdate says whether it may update
+	// itself; only then will it act on an update request (UpdateID). UpdateMsg is the outcome of the last requested update.
+	Version    string `json:"version,omitempty"`
+	AutoUpdate bool   `json:"autoUpdate,omitempty"`
+	UpdateID   string `json:"updateId,omitempty"` // set while an update requested by an administrator is pending
+	UpdateMsg  string `json:"updateMsg,omitempty"`
 }
 
 // Site is a bamboo agent: a private network that holds the real services.
@@ -54,8 +80,21 @@ type Site struct {
 	Local  bool   `json:"local"`
 	EdgeID string `json:"edgeId,omitempty"` // the edge a local site belongs to
 
+	Tags []string `json:"tags,omitempty"` // labels for grouping and filtering in the dashboard
+
+	// What the agent last said about its SSH server (see SiteReport.SSH and SSHRelay).
+	SSH      string `json:"ssh,omitempty"`
+	SSHRelay bool   `json:"sshRelay,omitempty"`
+
 	TunnelIP  string `json:"tunnelIp,omitempty"`  // the site's address inside the tunnel network
 	PublicKey string `json:"publicKey,omitempty"` // WireGuard public key, reported by the agent
+
+	// Version is the release the program reports (e.g. v0.1.5). AutoUpdate says whether it may update
+	// itself; only then will it act on an update request (UpdateID). UpdateMsg is the outcome of the last requested update.
+	Version    string `json:"version,omitempty"`
+	AutoUpdate bool   `json:"autoUpdate,omitempty"`
+	UpdateID   string `json:"updateId,omitempty"` // set while an update requested by an administrator is pending
+	UpdateMsg  string `json:"updateMsg,omitempty"`
 }
 
 // Domain is a base domain an org owns. Hostnames of web/tls mappings must sit under one.
@@ -111,10 +150,17 @@ type Mapping struct {
 	Hostname string   `json:"hostname,omitempty"` // web/tls only; "*.example.com" allowed
 	Targets  []Target `json:"targets"`            // udp uses only the first
 	Enabled  bool     `json:"enabled"`
+	Tags     []string `json:"tags,omitempty"` // labels for grouping and filtering in the dashboard
 	// ProxyProtocol ("v1" or "v2") makes the edge start each tcp/tls connection to the
 	// target with a PROXY protocol header carrying the real client address.
 	ProxyProtocol string `json:"proxyProtocol,omitempty"`
 
+	// A web mapping with RedirectTo set sends visitors there instead of to a target (so it has no
+	// targets, login or health checks). It answers 301 when RedirectPermanent, otherwise 302.
+	RedirectTo        string `json:"redirectTo,omitempty"`
+	RedirectPermanent bool   `json:"redirectPermanent,omitempty"`
+	RedirectKeepPath  bool   `json:"redirectKeepPath,omitempty"`
+
 	// web (http/https) options
 	Auth          Auth   `json:"auth"`
 	AuthEpoch     int64  `json:"authEpoch"` // bumped to sign every visitor out; set by hammerhead only
@@ -200,6 +246,58 @@ type MailCheck struct {
 	HeloResolves   bool              `json:"heloResolves"`   // the HELO name resolves back to the source IP
 	Port25         string            `json:"port25"`         // "open" or why not
 	DKIM           map[string]string `json:"dkim"`           // key ID -> "published", "missing" or "mismatch"
+
+	Domains    map[string]DomainCheck `json:"domains,omitempty"`    // sending domain -> its DNS checks
+	Blocklists []string               `json:"blocklists,omitempty"` // DNS blocklists that list the source IP
+	Listable   int                    `json:"listable,omitempty"`   // how many blocklists answered, so "none listed" means something
+	Outcomes   *MailOutcomes          `json:"outcomes,omitempty"`   // what happened to the mail this egress sent
+}
+
+// Finding statuses.
+const (
+	FindingPass = "pass"
+	FindingWarn = "warn"
+	FindingFail = "fail"
+	FindingNone = "none" // nothing published
+)
+
+// MailFinding is the result of one deliverability check, with what to do about it.
+type MailFinding struct {
+	Status string `json:"status"`
+	Detail string `json:"detail"`
+	Hint   string `json:"hint,omitempty"`
+}
+
+// DomainCheck is how the DNS of one sending domain looks to a receiving server.
+type DomainCheck struct {
+	SPF    MailFinding `json:"spf"`
+	DMARC  MailFinding `json:"dmarc"`
+	MTASTS MailFinding `json:"mtaSts"`
+}
+
+// MailOutcomes counts what became of the mail an egress handled over a window.
+type MailOutcomes struct {
+	Since     time.Time               `json:"since"`
+	Delivered int                     `json:"delivered"`
+	Deferred  int                     `json:"deferred"`           // 4xx: the receiver asked us to try later
+	Failed    int                     `json:"failed"`             // 5xx: refused for good
+	ByDomain  map[string]OutcomeCount `json:"byDomain,omitempty"` // recipient domain
+	Recent    []OutcomeFailure        `json:"recent,omitempty"`   // latest problems, newest first
+}
+
+// OutcomeCount is delivered/deferred/failed for one recipient domain.
+type OutcomeCount struct {
+	Delivered int `json:"delivered"`
+	Deferred  int `json:"deferred"`
+	Failed    int `json:"failed"`
+}
+
+// OutcomeFailure is one deferral or refusal, for the dashboard to explain.
+type OutcomeFailure struct {
+	Time   time.Time `json:"time"`
+	Domain string    `json:"domain"`
+	Code   int       `json:"code"`
+	Text   string    `json:"text"`
 }
 
 // ---------- edge <-> hammerhead ----------
@@ -216,6 +314,7 @@ type EdgeConfig struct {
 	Local      []string  `json:"local"`    // site IDs the edge dials directly on its own machine
 	AcmeEmail  string    `json:"acmeEmail"`
 	AcmeStages bool      `json:"acmeStaging"`
+	UpdateID   string    `json:"updateId,omitempty"` // an administrator asked this edge to update itself now; echoed back in its report
 
 	MailEgress []MailPolicy `json:"mailEgress"`
 }
@@ -245,6 +344,42 @@ type Stat struct {
 	Conns    int64 `json:"conns"`
 	BytesIn  int64 `json:"bytesIn"`  // client -> target
 	BytesOut int64 `json:"bytesOut"` // target -> client
+	// Sources are the busiest client addresses behind these numbers (at most MaxSources),
+	// busiest first. Addresses beyond that are only counted in the totals above.
+	Sources []SourceStat `json:"sources,omitempty"`
+}
+
+// MaxSources is how many client addresses an edge names per port in one report.
+const MaxSources = 10
+
+// SourceStat is one client address's traffic on a port since the last report.
+type SourceStat struct {
+	IP       string `json:"ip"`
+	Conns    int64  `json:"conns"`
+	BytesIn  int64  `json:"bytesIn"`
+	BytesOut int64  `json:"bytesOut"`
+}
+
+// TopSources keeps the n busiest of a port's client addresses (by bytes, then connections), busiest first.
+func TopSources(all map[string]*SourceStat, n int) []SourceStat {
+	out := make([]SourceStat, 0, len(all))
+	for _, s := range all {
+		out = append(out, *s)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		a, b := out[i], out[j]
+		if ta, tb := a.BytesIn+a.BytesOut, b.BytesIn+b.BytesOut; ta != tb {
+			return ta > tb
+		}
+		if a.Conns != b.Conns {
+			return a.Conns > b.Conns
+		}
+		return a.IP < b.IP
+	})
+	if len(out) > n {
+		out = out[:n]
+	}
+	return out
 }
 
 // LogEntry is one access-log line.
@@ -281,14 +416,37 @@ type ContainerPort struct {
 	Type    string `json:"type"`
 }
 
+// DiscoveredService is a listening service an agent found on its own network (local sockets or an
+// opt-in scan of configured subnets). Hammerhead offers these as suggested targets; nothing is
+// proxied until an administrator picks one.
+type DiscoveredService struct {
+	Host    string `json:"host"` // address to dial, as seen from the agent
+	Port    int    `json:"port"`
+	Proto   string `json:"proto"`             // "tcp"
+	Service string `json:"service,omitempty"` // best guess: http, https, ssh, smtp, ftp, mysql, ... (empty = unknown)
+	Detail  string `json:"detail,omitempty"`  // banner or Server header, trimmed
+	Source  string `json:"source"`            // "local" (listening socket on the agent's host) or "scan"
+}
+
 // EdgeReport is what blacktip posts back.
 type EdgeReport struct {
-	IPs        []string                 `json:"ips"`
+	IPs       []string                 `json:"ips"`
-	PublicKey  string                   `json:"publicKey"` // the edge's WireGuard public key
+	PublicKey string                   `json:"publicKey"` // the edge's WireGuard public key
-	Status     map[string]MappingStatus `json:"status"`
+	Status    map[string]MappingStatus `json:"status"`
-	Stats      map[string]Stat          `json:"stats"` // deltas since last report
+	Stats     map[string]Stat          `json:"stats"` // deltas since last report
-	Containers []Container              `json:"containers"`
+	// StatsSeconds is how long Stats covers (the time since the previous report that carried them),
-	MailChecks map[string]MailCheck     `json:"mailChecks"` // by mail-egress ID
+	// so a rate can be worked out however irregular the reports are.
+	StatsSeconds float64              `json:"statsSeconds,omitempty"`
+	Containers   []Container          `json:"containers"`
+	MailChecks   map[string]MailCheck `json:"mailChecks"` // by mail-egress ID
+	Version      string               `json:"version,omitempty"`
+	AutoUpdate   bool                 `json:"autoUpdate,omitempty"`
+	UpdateMsg    string               `json:"updateMsg,omitempty"` // outcome of the update asked for by UpdateID
+	UpdateID     string               `json:"updateId,omitempty"`
+	// progress of that update, for the dashboard's progress bar and log
+	UpdateStage string   `json:"updateStage,omitempty"`
+	UpdatePct   int      `json:"updatePct,omitempty"`
+	UpdateLog   []string `json:"updateLog,omitempty"`
 }
 
 // EdgeRef tells a site where to dial.
@@ -306,16 +464,35 @@ type SiteConfig struct {
 	SiteID   string    `json:"siteId"`
 	TunnelIP string    `json:"tunnelIp"` // this site's WireGuard address
 	Edges    []EdgeRef `json:"edges"`
+	UpdateID string    `json:"updateId,omitempty"` // an administrator asked this agent to update itself now; echoed back in its report
+
+	// Terminals lists terminal sessions waiting for this agent. For each, the agent opens a connection
+	// to hammerhead at /site/terminal/<id> and joins it to its own SSH server, so a shell can be opened
+	// from the dashboard without any port being open on an edge.
+	Terminals []string `json:"terminals,omitempty"`
 
 	MailEgress []MailSpec `json:"mailEgress"`
 }
 
 // SiteReport is what bamboo posts back.
 type SiteReport struct {
-	PublicKey  string            `json:"publicKey"`  // the agent's WireGuard public key
+	PublicKey  string              `json:"publicKey"`  // the agent's WireGuard public key
-	MailEgress map[string]string `json:"mailEgress"` // egress ID -> "" (listening) or an error
+	MailEgress map[string]string   `json:"mailEgress"` // egress ID -> "" (listening) or an error
-	Connected  []string          `json:"connected"`
+	Connected  []string            `json:"connected"`
-	Containers []Container       `json:"containers"`
+	Containers []Container         `json:"containers"`
+	Discovered []DiscoveredService `json:"discovered,omitempty"`
+	Version    string              `json:"version,omitempty"`
+	AutoUpdate bool                `json:"autoUpdate,omitempty"`
+	UpdateMsg  string              `json:"updateMsg,omitempty"`
+	UpdateID   string              `json:"updateId,omitempty"`
+	// SSH is where this machine's SSH server listens ("" when it is not running). SSHRelay says this
+	// agent can answer SiteConfig.Terminals, which is how an agent that is too old to relay shows itself.
+	SSH      string `json:"ssh,omitempty"`
+	SSHRelay bool   `json:"sshRelay,omitempty"`
+	// progress of that update, for the dashboard's progress bar and log
+	UpdateStage string   `json:"updateStage,omitempty"`
+	UpdatePct   int      `json:"updatePct,omitempty"`
+	UpdateLog   []string `json:"updateLog,omitempty"`
 }
 
 // HostMatches reports whether host matches an exact name or a "*.example.com" pattern.

model/model_test.go

@@ -0,0 +1,41 @@
+package model
+
+import "testing"
+
+func TestRedirectLocation(t *testing.T) {
+	cases := []struct {
+		m    Mapping
+		uri  string
+		want string
+	}{
+		{Mapping{RedirectTo: "https://new.example.com"}, "/a?b=1", "https://new.example.com"},
+		{Mapping{RedirectTo: "https://new.example.com", RedirectKeepPath: true}, "/a?b=1", "https://new.example.com/a?b=1"},
+		{Mapping{RedirectTo: "https://new.example.com/", RedirectKeepPath: true}, "/", "https://new.example.com/"},
+		{Mapping{RedirectTo: "https://new.example.com/base", RedirectKeepPath: true}, "/x/y", "https://new.example.com/base/x/y"},
+		{Mapping{RedirectTo: "https://new.example.com", RedirectKeepPath: true}, "", "https://new.example.com/"},
+	}
+	for _, c := range cases {
+		if got := c.m.RedirectLocation(c.uri); got != c.want {
+			t.Errorf("%+v %q: got %q, want %q", c.m, c.uri, got, c.want)
+		}
+	}
+	if (Mapping{}).IsRedirect() || !(Mapping{RedirectTo: "https://x"}).IsRedirect() {
+		t.Error("IsRedirect is wrong")
+	}
+}
+
+func TestTopSources(t *testing.T) {
+	all := map[string]*SourceStat{
+		"a": {IP: "a", Conns: 1, BytesIn: 10, BytesOut: 5},
+		"b": {IP: "b", Conns: 9, BytesIn: 1},
+		"c": {IP: "c", Conns: 2, BytesIn: 100},
+		"d": {IP: "d", Conns: 3, BytesIn: 1},
+	}
+	got := TopSources(all, 3)
+	if len(got) != 3 || got[0].IP != "c" || got[1].IP != "a" || got[2].IP != "b" {
+		t.Fatalf("order: %+v", got) // c most bytes, a next, then b beats d on connections
+	}
+	if got := TopSources(all, 10); len(got) != 4 {
+		t.Fatalf("all: %+v", got)
+	}
+}

secretbox/secretbox.go

@@ -0,0 +1,87 @@
+// Package secretbox encrypts small secrets (tokens, passwords, private keys)
+// before they are written to the database, using AES-256-GCM.
+//
+// Sealed values look like "enc1:<base64>". Opening a value that isn't sealed
+// returns it unchanged, so data written before encryption was turned on is
+// still readable and gets sealed the next time it is saved.
+package secretbox
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"strings"
+)
+
+const prefix = "enc1:"
+
+type Box struct{ aead cipher.AEAD }
+
+// GenerateKey returns a new random key in the form New accepts.
+func GenerateKey() string {
+	b := make([]byte, 32)
+	rand.Read(b)
+	return base64.StdEncoding.EncodeToString(b)
+}
+
+// New builds a Box from a key given as base64 or hex of 32 bytes. Any other
+// text is treated as a passphrase and stretched with SHA-256.
+func New(key string) (*Box, error) {
+	key = strings.TrimSpace(key)
+	if key == "" {
+		return nil, errors.New("empty key")
+	}
+	raw := decodeKey(key)
+	block, err := aes.NewCipher(raw)
+	if err != nil {
+		return nil, err
+	}
+	aead, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	return &Box{aead: aead}, nil
+}
+
+func decodeKey(key string) []byte {
+	if b, err := base64.StdEncoding.DecodeString(key); err == nil && len(b) == 32 {
+		return b
+	}
+	if b, err := hex.DecodeString(key); err == nil && len(b) == 32 {
+		return b
+	}
+	sum := sha256.Sum256([]byte(key))
+	return sum[:]
+}
+
+// Seal encrypts s. The empty string stays empty so unset secrets stay visibly unset.
+func (b *Box) Seal(s string) string {
+	if s == "" || strings.HasPrefix(s, prefix) {
+		return s
+	}
+	nonce := make([]byte, b.aead.NonceSize())
+	rand.Read(nonce)
+	return prefix + base64.StdEncoding.EncodeToString(b.aead.Seal(nonce, nonce, []byte(s), nil))
+}
+
+// Open decrypts a sealed value. A wrong key or damaged data is an error, so a
+// bad key can never silently turn secrets into garbage.
+func (b *Box) Open(s string) (string, error) {
+	if !strings.HasPrefix(s, prefix) {
+		return s, nil
+	}
+	raw, err := base64.StdEncoding.DecodeString(s[len(prefix):])
+	if err != nil || len(raw) < b.aead.NonceSize() {
+		return "", errors.New("damaged secret")
+	}
+	n := b.aead.NonceSize()
+	plain, err := b.aead.Open(nil, raw[:n], raw[n:], nil)
+	if err != nil {
+		return "", errors.New("can't decrypt a stored secret: is HAMMERHEAD_MASTER_KEY the key it was saved with?")
+	}
+	return string(plain), nil
+}

secretbox/secretbox_test.go

@@ -0,0 +1,51 @@
+package secretbox
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestSealOpenRoundTripAndKeys(t *testing.T) {
+	key := GenerateKey()
+	b, err := New(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sealed := b.Seal("hunter2")
+	if !strings.HasPrefix(sealed, "enc1:") || strings.Contains(sealed, "hunter2") {
+		t.Fatalf("not sealed: %q", sealed)
+	}
+	if again := b.Seal("hunter2"); again == sealed {
+		t.Error("two seals of the same value must differ (random nonce)")
+	}
+	if b.Seal(sealed) != sealed {
+		t.Error("sealing an already sealed value must not double-encrypt")
+	}
+	if got, err := b.Open(sealed); err != nil || got != "hunter2" {
+		t.Errorf("Open = %q, %v", got, err)
+	}
+	if got, _ := b.Open("legacy plaintext"); got != "legacy plaintext" {
+		t.Errorf("unsealed legacy data should pass through, got %q", got)
+	}
+	if b.Seal("") != "" {
+		t.Error("empty stays empty")
+	}
+
+	other, _ := New(GenerateKey())
+	if _, err := other.Open(sealed); err == nil {
+		t.Error("the wrong key must fail loudly, not return garbage")
+	}
+	if _, err := b.Open(sealed[:len(sealed)-4] + "AAAA"); err == nil {
+		t.Error("tampered data must fail")
+	}
+
+	// a passphrase works as a key, and is stable
+	p1, _ := New("correct horse battery staple")
+	p2, _ := New("correct horse battery staple")
+	if got, err := p2.Open(p1.Seal("x")); err != nil || got != "x" {
+		t.Errorf("passphrase keys must be reproducible: %q %v", got, err)
+	}
+	if _, err := New("  "); err == nil {
+		t.Error("an empty key must be rejected")
+	}
+}

selfupdate/restart_unix.go

@@ -7,7 +7,7 @@ import (
 	"syscall"
 )
 
-// restart replaces this process with the new binary, with the same arguments and environment.
+// Restart replaces this process with the new binary, with the same arguments and environment.
-func restart(exe string) error {
+func Restart(exe string) error {
 	return syscall.Exec(exe, os.Args, os.Environ())
 }

selfupdate/restart_windows.go

@@ -2,6 +2,6 @@ package selfupdate
 
 import "errors"
 
-func restart(string) error {
+func Restart(string) error {
 	return errors.New("restarting in place is not supported on Windows")
 }

selfupdate/selfupdate.go

@@ -29,6 +29,15 @@ import (
 
 const maxArchive = 300 << 20
 
+// Event is one step of an update, for showing progress: the stage ("checking", "downloading",
+// "verifying", "installing", "restarting", "done" or "failed"), how far along it is (0-100, or -1 to
+// leave the bar where it was) and, when there is something worth showing in a log, a line of text.
+type Event struct {
+	Stage string
+	Pct   int
+	Line  string
+}
+
 // Config says what to update and where from.
 type Config struct {
 	Server  string // e.g. https://git.veillax.com
@@ -38,6 +47,8 @@ type Config struct {
 	// Interval between checks (default 6 hours).
 	Interval time.Duration
 	Logf     func(format string, args ...any)
+	// Progress, if set, is told about every step of an update (it is called from the updating goroutine).
+	Progress func(Event)
 
 	// The fields below exist for tests and unusual setups.
 	HTTP    *http.Client
@@ -72,7 +83,7 @@ func (c *Config) defaults() {
 		c.Verify = verifyRuns(c.App)
 	}
 	if c.Restart == nil {
-		c.Restart = restart
+		c.Restart = Restart
 	}
 	c.Server = strings.TrimRight(c.Server, "/")
 }
@@ -103,35 +114,40 @@ func Run(ctx context.Context, cfg Config) {
 	}
 }
 
+func (c Config) say(stage string, pct int, line string) {
+	if c.Progress != nil {
+		c.Progress(Event{Stage: stage, Pct: pct, Line: line})
+	}
+}
+
 // CheckOnce looks for a newer release and installs it. It returns the new tag when it
 // installed one (and Restart returned, which only happens in tests or on failure).
 func CheckOnce(ctx context.Context, cfg Config) (string, error) {
 	cfg.defaults()
+	tag, err := checkOnce(ctx, cfg)
+	if err != nil {
+		cfg.say("failed", -1, "Failed: "+err.Error())
+	}
+	return tag, err
+}
+
+func checkOnce(ctx context.Context, cfg Config) (string, error) {
 	cur, ok := parseVersion(cfg.Current)
 	if !ok {
 		return "", fmt.Errorf("%q is not a release version", cfg.Current)
 	}
-	var rel struct {
+	cfg.say("checking", 2, "Checking "+cfg.Server+" for the newest release of "+cfg.App)
-		Tag        string `json:"tag_name"`
+	rel, err := latestRelease(ctx, cfg)
-		Prerelease bool   `json:"prerelease"`
-		Draft      bool   `json:"draft"`
-		Assets     []struct {
-			Name string `json:"name"`
-			URL  string `json:"browser_download_url"`
-		} `json:"assets"`
-	}
-	body, err := get(ctx, cfg.HTTP, cfg.Server+"/api/v1/repos/"+cfg.Repo+"/releases/latest", 1<<20)
 	if err != nil {
-		return "", fmt.Errorf("checking for a new release: %w", err)
+		return "", err
-	}
-	if err := json.Unmarshal(body, &rel); err != nil {
-		return "", fmt.Errorf("reading the latest release: %w", err)
 	}
 	if rel.Prerelease || rel.Draft {
+		cfg.say("done", 100, "Already on the newest release ("+cfg.Current+")")
 		return "", nil
 	}
 	latest, ok := parseVersion(rel.Tag)
 	if !ok || !latest.newerThan(cur) {
+		cfg.say("done", 100, "Already on the newest release ("+cfg.Current+")")
 		return "", nil // up to date, or something we don't understand: never downgrade
 	}
 
@@ -152,8 +168,9 @@ func CheckOnce(ctx context.Context, cfg Config) (string, error) {
 		return "", fmt.Errorf("release %s has no %s with SHA256SUMS yet (it may still be building); will retry", rel.Tag, archive)
 	}
 	cfg.Logf("auto-update: %s is available (running %s); downloading %s", rel.Tag, cfg.Current, archive)
+	cfg.say("checking", 5, rel.Tag+" is available (running "+cfg.Current+")")
 
-	sums, err := get(ctx, cfg.HTTP, sumsURL, 1<<20)
+	sums, err := get(ctx, cfg.HTTP, sumsURL, 1<<20, nil)
 	if err != nil {
 		return "", err
 	}
@@ -167,14 +184,31 @@ func CheckOnce(ctx context.Context, cfg Config) (string, error) {
 	if want == "" {
 		return "", fmt.Errorf("SHA256SUMS does not list %s", archive)
 	}
-	data, err := get(ctx, cfg.HTTP, archiveURL, maxArchive)
+	cfg.say("downloading", 6, "Downloading "+archive)
+	lastQuarter := 0
+	data, err := get(ctx, cfg.HTTP, archiveURL, maxArchive, func(done, total int64) {
+		if total <= 0 {
+			cfg.say("downloading", -1, "")
+			return
+		}
+		pct := 6 + int(80*done/total)
+		line := ""
+		if q := int(100 * done / total / 25); q > lastQuarter {
+			lastQuarter = q
+			line = fmt.Sprintf("Downloaded %.1f of %.1f MB (%d%%)", float64(done)/1e6, float64(total)/1e6, 100*done/total)
+		}
+		cfg.say("downloading", pct, line)
+	})
 	if err != nil {
 		return "", err
 	}
+	cfg.say("verifying", 88, "Verifying the checksum")
 	got := sha256.Sum256(data)
 	if hex.EncodeToString(got[:]) != want {
 		return "", fmt.Errorf("checksum mismatch for %s: not installing", archive)
 	}
+	cfg.say("verifying", 90, "The checksum matches")
+	cfg.say("installing", 92, "Unpacking the new binary")
 	bin, err := extract(data, cfg.App)
 	if err != nil {
 		return "", err
@@ -193,22 +227,66 @@ func CheckOnce(ctx context.Context, cfg Config) (string, error) {
 	if err := os.WriteFile(tmp, bin, 0o755); err != nil {
 		return "", fmt.Errorf("can't write next to %s (is its folder writable by this user?): %w", exe, err)
 	}
+	cfg.say("installing", 94, "Checking that the new binary starts")
 	if err := cfg.Verify(tmp, rel.Tag); err != nil {
 		os.Remove(tmp)
 		return "", fmt.Errorf("the new binary failed its check, keeping the old one: %w", err)
 	}
+	cfg.say("installing", 97, "Replacing "+exe)
 	if err := os.Rename(tmp, exe); err != nil { // atomic; the running process keeps its old file
 		os.Remove(tmp)
 		return "", fmt.Errorf("replacing %s: %w", exe, err)
 	}
 	cfg.Logf("auto-update: installed %s, restarting", rel.Tag)
+	cfg.say("restarting", 100, "Installed "+rel.Tag+"; restarting into it")
 	if err := cfg.Restart(exe); err != nil {
 		return rel.Tag, fmt.Errorf("installed %s but could not restart into it (it will start with the next restart): %w", rel.Tag, err)
 	}
 	return rel.Tag, nil
 }
 
-func get(ctx context.Context, c *http.Client, url string, max int64) ([]byte, error) {
+type release struct {
+	Tag        string `json:"tag_name"`
+	Prerelease bool   `json:"prerelease"`
+	Draft      bool   `json:"draft"`
+	Assets     []struct {
+		Name string `json:"name"`
+		URL  string `json:"browser_download_url"`
+	} `json:"assets"`
+}
+
+func latestRelease(ctx context.Context, cfg Config) (release, error) {
+	var rel release
+	body, err := get(ctx, cfg.HTTP, cfg.Server+"/api/v1/repos/"+cfg.Repo+"/releases/latest", 1<<20, nil)
+	if err != nil {
+		return rel, fmt.Errorf("checking for a new release: %w", err)
+	}
+	if err := json.Unmarshal(body, &rel); err != nil {
+		return rel, fmt.Errorf("reading the latest release: %w", err)
+	}
+	return rel, nil
+}
+
+// Check reports the newest full release and whether it is newer than cfg.Current, without
+// installing anything. A development build (a Current that isn't vX.Y.Z) is never "newer".
+func Check(ctx context.Context, cfg Config) (latest string, newer bool, err error) {
+	cfg.defaults()
+	rel, err := latestRelease(ctx, cfg)
+	if err != nil {
+		return "", false, err
+	}
+	if rel.Prerelease || rel.Draft {
+		return "", false, nil
+	}
+	l, ok := parseVersion(rel.Tag)
+	if !ok {
+		return rel.Tag, false, nil
+	}
+	cur, ok := parseVersion(cfg.Current)
+	return rel.Tag, ok && l.newerThan(cur), nil
+}
+
+func get(ctx context.Context, c *http.Client, url string, max int64, onProgress func(done, total int64)) ([]byte, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
 		return nil, err
@@ -222,12 +300,23 @@ func get(ctx context.Context, c *http.Client, url string, max int64) ([]byte, er
 	if resp.StatusCode != 200 {
 		return nil, fmt.Errorf("%s: %s", url, resp.Status)
 	}
-	b, err := io.ReadAll(io.LimitReader(resp.Body, max+1))
+	var b []byte
-	if err != nil {
+	buf := make([]byte, 64<<10)
-		return nil, err
+	for {
-	}
+		n, err := resp.Body.Read(buf)
-	if int64(len(b)) > max {
+		b = append(b, buf[:n]...)
-		return nil, fmt.Errorf("%s is larger than %d bytes", url, max)
+		if int64(len(b)) > max {
+			return nil, fmt.Errorf("%s is larger than %d bytes", url, max)
+		}
+		if onProgress != nil && n > 0 {
+			onProgress(int64(len(b)), resp.ContentLength)
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
 	}
 	return b, nil
 }

selfupdate/selfupdate_test.go

@@ -135,3 +135,63 @@ func TestVersionOrdering(t *testing.T) {
 		}
 	}
 }
+
+func TestCheckReportsWithoutInstalling(t *testing.T) {
+	srv := fakeGitea(t, "v0.2.0", false, false)
+	for _, c := range []struct {
+		current string
+		newer   bool
+	}{{"v0.1.0", true}, {"v0.2.0", false}, {"v0.3.0", false}, {"dev", false}, {"v0.1.0-local", false}} {
+		cfg, exe, restarted := setup(t, srv, c.current)
+		tag, newer, err := Check(context.Background(), cfg)
+		if err != nil || tag != "v0.2.0" || newer != c.newer {
+			t.Errorf("%s: tag=%q newer=%v err=%v", c.current, tag, newer, err)
+		}
+		if b, _ := os.ReadFile(exe); string(b) != "OLD-BINARY" || *restarted != "" {
+			t.Errorf("%s: Check changed things", c.current)
+		}
+	}
+}
+
+func TestReportsProgressThroughTheWholeUpdate(t *testing.T) {
+	cfg, _, _ := setup(t, fakeGitea(t, "v0.2.0", false, false), "v0.1.0")
+	var events []Event
+	cfg.Progress = func(e Event) { events = append(events, e) }
+	if _, err := CheckOnce(context.Background(), cfg); err != nil {
+		t.Fatal(err)
+	}
+	var stages []string
+	last, lines := 0, ""
+	for _, e := range events {
+		if len(stages) == 0 || stages[len(stages)-1] != e.Stage {
+			stages = append(stages, e.Stage)
+		}
+		if e.Pct >= 0 {
+			if e.Pct < last {
+				t.Errorf("progress went backwards: %d after %d (%+v)", e.Pct, last, e)
+			}
+			last = e.Pct
+		}
+		lines += e.Line + "\n"
+	}
+	if got := strings.Join(stages, ","); got != "checking,downloading,verifying,installing,restarting" {
+		t.Errorf("stages: %s", got)
+	}
+	if last != 100 || !strings.Contains(lines, "v0.2.0 is available") || !strings.Contains(lines, "checksum matches") {
+		t.Errorf("progress ended at %d with log:\n%s", last, lines)
+	}
+}
+
+func TestProgressReportsFailureAndNothingToDo(t *testing.T) {
+	cfg, _, _ := setup(t, fakeGitea(t, "v0.2.0", false, true), "v0.1.0")
+	var last Event
+	cfg.Progress = func(e Event) { last = e }
+	if _, err := CheckOnce(context.Background(), cfg); err == nil || last.Stage != "failed" || !strings.Contains(last.Line, "checksum") {
+		t.Errorf("err=%v last=%+v", err, last)
+	}
+	cfg, _, _ = setup(t, fakeGitea(t, "v0.2.0", false, false), "v0.2.0")
+	cfg.Progress = func(e Event) { last = e }
+	if tag, err := CheckOnce(context.Background(), cfg); err != nil || tag != "" || last.Stage != "done" || last.Pct != 100 {
+		t.Errorf("tag=%q err=%v last=%+v", tag, err, last)
+	}
+}