Merge branch 'feature/discovered-services'

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.gitea/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Formatting
+        run: |
+          unformatted=$(gofmt -l .)
+          if [ -n "$unformatted" ]; then
+            echo "These files need gofmt:"
+            echo "$unformatted"
+            exit 1
+          fi
+
+      - name: Vet
+        run: go vet ./...
+
+      - name: Test
+        run: go test ./...
+
+      # Reef has platform-specific files (Windows named pipes for Docker), so make
+      # sure it builds everywhere the programs that import it do.
+      - name: Builds on every platform
+        run: |
+          for t in linux/amd64 linux/arm64 linux/arm windows/amd64 darwin/amd64 darwin/arm64; do
+            echo "building for $t"
+            GOOS=${t%/*} GOARCH=${t#*/} GOARM=7 CGO_ENABLED=0 go build ./...
+          done

README.md

@@ -1,6 +1,6 @@
 # Reef
 
-A shared library for Hammerhead, Blacktip, and Bamboo
+A shared library for Hammerhead, Blacktip, Bamboo and Sandbar
 
 Shared types and building blocks for the sharks stack: [Hammerhead](https://git.veillax.com/Veillax/hammerhead) (control plane), [Blacktip](https://git.veillax.com/Veillax/blacktip) (edge) and [Bamboo](https://git.veillax.com/Veillax/bamboo) (site agent).
 
@@ -15,6 +15,7 @@ Reef holds the code all three programs agree on, so each can be built and releas
 | `netguard` | Refuses internal addresses for anything that connects out on a user's behalf (webhooks, mail delivery), and dials the address it checked |
 | `docker` | Lists running containers through the Docker Engine API, with no SDK |
 | `dotenv` | Loads a `.env` file into the environment without overriding real variables |
+| `secretbox` | AES-256-GCM sealing of small secrets (`enc1:` values) before they are written to a database |
 
 ## Using it
 

dmarc/dmarc.go

@@ -0,0 +1,274 @@
+// Package dmarc parses DMARC aggregate (rua) reports (RFC 7489 appendix A).
+// Receivers send them as XML, usually gzip or zip compressed, usually as an
+// email attachment; Parse accepts any of those forms.
+package dmarc
+
+import (
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"encoding/xml"
+	"errors"
+	"io"
+	"mime"
+	"mime/multipart"
+	"mime/quotedprintable"
+	"net/mail"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// maxXML caps a decompressed report; aggregate reports are small, so a larger one is a zip bomb.
+const maxXML = 32 << 20
+
+// Report is one receiver's summary of the mail it saw claiming to be from Domain.
+type Report struct {
+	Org      string    `json:"org"` // the receiver that sent the report
+	Email    string    `json:"email"`
+	ReportID string    `json:"reportId"`
+	Begin    time.Time `json:"begin"`
+	End      time.Time `json:"end"`
+	Domain   string    `json:"domain"`
+	Policy   Policy    `json:"policy"`
+	Records  []Record  `json:"records"`
+}
+
+// Policy is the DMARC policy the receiver applied.
+type Policy struct {
+	P     string `json:"p"`   // none, quarantine or reject
+	SP    string `json:"sp"`  // for subdomains
+	Pct   int    `json:"pct"` // percent of mail the policy applies to
+	ADKIM string `json:"adkim"`
+	ASPF  string `json:"aspf"`
+}
+
+// Record is one row: a sending IP and what happened to its mail.
+type Record struct {
+	SourceIP     string       `json:"sourceIp"`
+	Count        int          `json:"count"`
+	Disposition  string       `json:"disposition"` // what the receiver did: none, quarantine, reject
+	DKIM         string       `json:"dkim"`        // aligned DKIM result: pass or fail
+	SPF          string       `json:"spf"`         // aligned SPF result: pass or fail
+	HeaderFrom   string       `json:"headerFrom"`
+	EnvelopeFrom string       `json:"envelopeFrom,omitempty"`
+	DKIMAuth     []AuthResult `json:"dkimAuth,omitempty"`
+	SPFAuth      []AuthResult `json:"spfAuth,omitempty"`
+}
+
+// AuthResult is a raw DKIM or SPF verdict, before alignment with the From domain.
+type AuthResult struct {
+	Domain   string `json:"domain"`
+	Selector string `json:"selector,omitempty"`
+	Result   string `json:"result"`
+}
+
+// Passed reports whether the row's mail passed DMARC: either aligned check passing is enough.
+func (r Record) Passed() bool { return r.DKIM == "pass" || r.SPF == "pass" }
+
+type xmlFeedback struct {
+	Meta struct {
+		Org      string `xml:"org_name"`
+		Email    string `xml:"email"`
+		ReportID string `xml:"report_id"`
+		Range    struct {
+			Begin int64 `xml:"begin"`
+			End   int64 `xml:"end"`
+		} `xml:"date_range"`
+	} `xml:"report_metadata"`
+	Policy struct {
+		Domain string `xml:"domain"`
+		ADKIM  string `xml:"adkim"`
+		ASPF   string `xml:"aspf"`
+		P      string `xml:"p"`
+		SP     string `xml:"sp"`
+		Pct    string `xml:"pct"`
+	} `xml:"policy_published"`
+	Records []struct {
+		Row struct {
+			SourceIP string `xml:"source_ip"`
+			Count    int    `xml:"count"`
+			Eval     struct {
+				Disposition string `xml:"disposition"`
+				DKIM        string `xml:"dkim"`
+				SPF         string `xml:"spf"`
+			} `xml:"policy_evaluated"`
+		} `xml:"row"`
+		Identifiers struct {
+			HeaderFrom   string `xml:"header_from"`
+			EnvelopeFrom string `xml:"envelope_from"`
+		} `xml:"identifiers"`
+		Auth struct {
+			DKIM []struct {
+				Domain   string `xml:"domain"`
+				Selector string `xml:"selector"`
+				Result   string `xml:"result"`
+			} `xml:"dkim"`
+			SPF []struct {
+				Domain string `xml:"domain"`
+				Result string `xml:"result"`
+			} `xml:"spf"`
+		} `xml:"auth_results"`
+	} `xml:"record"`
+}
+
+// Parse reads a report from raw XML, gzip, zip, or a whole email carrying one of those.
+func Parse(data []byte) (*Report, error) {
+	xmlData, err := unwrap(data, 0)
+	if err != nil {
+		return nil, err
+	}
+	var f xmlFeedback
+	if err := xml.Unmarshal(xmlData, &f); err != nil {
+		return nil, errors.New("not a DMARC report: " + err.Error())
+	}
+	if f.Policy.Domain == "" {
+		return nil, errors.New("not a DMARC report: no published policy domain")
+	}
+	if strings.TrimSpace(f.Meta.ReportID) == "" {
+		return nil, errors.New("not a DMARC report: no report_id")
+	}
+	pct := 100
+	if n, err := strconv.Atoi(strings.TrimSpace(f.Policy.Pct)); err == nil && n >= 0 && n <= 100 {
+		pct = n
+	}
+	r := &Report{
+		Org: strings.TrimSpace(f.Meta.Org), Email: strings.TrimSpace(f.Meta.Email), ReportID: strings.TrimSpace(f.Meta.ReportID),
+		Begin:  time.Unix(f.Meta.Range.Begin, 0).UTC(),
+		End:    time.Unix(f.Meta.Range.End, 0).UTC(),
+		Domain: lower(f.Policy.Domain),
+		Policy: Policy{P: lower(f.Policy.P), SP: lower(f.Policy.SP), Pct: pct, ADKIM: lower(f.Policy.ADKIM), ASPF: lower(f.Policy.ASPF)},
+	}
+	for _, x := range f.Records {
+		rec := Record{
+			SourceIP: strings.TrimSpace(x.Row.SourceIP), Count: x.Row.Count,
+			Disposition: lower(x.Row.Eval.Disposition), DKIM: lower(x.Row.Eval.DKIM), SPF: lower(x.Row.Eval.SPF),
+			HeaderFrom: lower(x.Identifiers.HeaderFrom), EnvelopeFrom: lower(x.Identifiers.EnvelopeFrom),
+		}
+		for _, d := range x.Auth.DKIM {
+			rec.DKIMAuth = append(rec.DKIMAuth, AuthResult{Domain: lower(d.Domain), Selector: d.Selector, Result: lower(d.Result)})
+		}
+		for _, s := range x.Auth.SPF {
+			rec.SPFAuth = append(rec.SPFAuth, AuthResult{Domain: lower(s.Domain), Result: lower(s.Result)})
+		}
+		r.Records = append(r.Records, rec)
+	}
+	return r, nil
+}
+
+func lower(s string) string { return strings.ToLower(strings.TrimSpace(s)) }
+
+// unwrap peels gzip, zip and email layers until XML is left.
+func unwrap(data []byte, depth int) ([]byte, error) {
+	if depth > 4 {
+		return nil, errors.New("report is wrapped too deeply")
+	}
+	switch {
+	case bytes.HasPrefix(data, []byte{0x1f, 0x8b}):
+		zr, err := gzip.NewReader(bytes.NewReader(data))
+		if err != nil {
+			return nil, err
+		}
+		defer zr.Close()
+		return readLimited(zr, depth)
+	case bytes.HasPrefix(data, []byte("PK\x03\x04")):
+		zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+		if err != nil {
+			return nil, err
+		}
+		for _, f := range zr.File {
+			if f.FileInfo().IsDir() {
+				continue
+			}
+			rc, err := f.Open()
+			if err != nil {
+				return nil, err
+			}
+			out, err := readLimited(rc, depth)
+			rc.Close()
+			return out, err
+		}
+		return nil, errors.New("empty zip")
+	}
+	trimmed := bytes.TrimSpace(data)
+	if bytes.HasPrefix(trimmed, []byte("<")) {
+		return trimmed, nil
+	}
+	return fromEmail(data, depth)
+}
+
+func readLimited(r io.Reader, depth int) ([]byte, error) {
+	b, err := io.ReadAll(io.LimitReader(r, maxXML+1))
+	if err != nil {
+		return nil, err
+	}
+	if len(b) > maxXML {
+		return nil, errors.New("report is too large")
+	}
+	return unwrap(b, depth+1)
+}
+
+// fromEmail finds the report attachment in a message.
+func fromEmail(data []byte, depth int) ([]byte, error) {
+	m, err := mail.ReadMessage(bytes.NewReader(data))
+	if err != nil {
+		return nil, errors.New("not a DMARC report: unrecognised format")
+	}
+	b, err := fromPart(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body, depth)
+	if err != nil {
+		return nil, err
+	}
+	if b == nil {
+		return nil, errors.New("no report attached to the email")
+	}
+	return b, nil
+}
+
+func fromPart(contentType, encoding string, body io.Reader, depth int) ([]byte, error) {
+	mt, params, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		mt = "text/plain"
+	}
+	if strings.HasPrefix(mt, "multipart/") {
+		mr := multipart.NewReader(body, params["boundary"])
+		for {
+			p, err := mr.NextPart()
+			if err != nil {
+				return nil, nil
+			}
+			b, err := fromPart(p.Header.Get("Content-Type"), p.Header.Get("Content-Transfer-Encoding"), p, depth)
+			if err != nil || b != nil {
+				return b, err
+			}
+		}
+	}
+	switch mt {
+	case "application/gzip", "application/x-gzip", "application/zip", "application/x-zip-compressed", "application/xml", "text/xml", "application/octet-stream":
+		raw, err := io.ReadAll(io.LimitReader(decodeBody(encoding, body), maxXML+1))
+		if err != nil {
+			return nil, err
+		}
+		return unwrap(raw, depth+1)
+	}
+	return nil, nil
+}
+
+func decodeBody(encoding string, r io.Reader) io.Reader {
+	switch strings.ToLower(strings.TrimSpace(encoding)) {
+	case "base64":
+		// mail wraps base64 in lines, which the decoder ignores when it is given the line breaks stripped
+		b, _ := io.ReadAll(io.LimitReader(r, maxXML*2))
+		clean := strings.Map(func(c rune) rune {
+			if c == '\r' || c == '\n' || c == ' ' || c == '\t' {
+				return -1
+			}
+			return c
+		}, string(b))
+		out, _ := base64.StdEncoding.DecodeString(clean)
+		return bytes.NewReader(out)
+	case "quoted-printable":
+		return quotedprintable.NewReader(r)
+	}
+	return r
+}

dmarc/dmarc_test.go

@@ -0,0 +1,157 @@
+package dmarc
+
+import (
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"strings"
+	"testing"
+)
+
+const sample = `<?xml version="1.0" encoding="UTF-8" ?>
+<feedback>
+  <report_metadata>
+    <org_name>google.com</org_name>
+    <email>noreply-dmarc-support@google.com</email>
+    <report_id>12345</report_id>
+    <date_range><begin>1700000000</begin><end>1700086399</end></date_range>
+  </report_metadata>
+  <policy_published>
+    <domain>Example.COM</domain>
+    <adkim>r</adkim><aspf>r</aspf><p>none</p><sp>none</sp><pct>100</pct>
+  </policy_published>
+  <record>
+    <row>
+      <source_ip>203.0.113.7</source_ip><count>12</count>
+      <policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>fail</spf></policy_evaluated>
+    </row>
+    <identifiers><header_from>example.com</header_from><envelope_from>bounce.example.com</envelope_from></identifiers>
+    <auth_results>
+      <dkim><domain>example.com</domain><selector>s1</selector><result>pass</result></dkim>
+      <spf><domain>bounce.example.com</domain><result>pass</result></spf>
+    </auth_results>
+  </record>
+  <record>
+    <row>
+      <source_ip>198.51.100.9</source_ip><count>3</count>
+      <policy_evaluated><disposition>quarantine</disposition><dkim>fail</dkim><spf>fail</spf></policy_evaluated>
+    </row>
+    <identifiers><header_from>example.com</header_from></identifiers>
+    <auth_results>
+      <spf><domain>example.com</domain><result>softfail</result></spf>
+    </auth_results>
+  </record>
+</feedback>`
+
+func check(t *testing.T, r *Report) {
+	t.Helper()
+	if r.Org != "google.com" || r.ReportID != "12345" || r.Domain != "example.com" {
+		t.Fatalf("metadata: %+v", r)
+	}
+	if r.Begin.Unix() != 1700000000 || r.End.Unix() != 1700086399 {
+		t.Fatalf("range: %v %v", r.Begin, r.End)
+	}
+	if r.Policy.P != "none" || r.Policy.Pct != 100 || r.Policy.ADKIM != "r" {
+		t.Fatalf("policy: %+v", r.Policy)
+	}
+	if len(r.Records) != 2 {
+		t.Fatalf("records: %d", len(r.Records))
+	}
+	a, b := r.Records[0], r.Records[1]
+	if a.SourceIP != "203.0.113.7" || a.Count != 12 || !a.Passed() || a.EnvelopeFrom != "bounce.example.com" {
+		t.Fatalf("first record: %+v", a)
+	}
+	if len(a.DKIMAuth) != 1 || a.DKIMAuth[0].Selector != "s1" || len(a.SPFAuth) != 1 {
+		t.Fatalf("auth results: %+v", a)
+	}
+	if b.Passed() || b.Disposition != "quarantine" || b.SPFAuth[0].Result != "softfail" {
+		t.Fatalf("second record: %+v", b)
+	}
+}
+
+func gz(t *testing.T, s string) []byte {
+	var buf bytes.Buffer
+	w := gzip.NewWriter(&buf)
+	w.Write([]byte(s))
+	w.Close()
+	return buf.Bytes()
+}
+
+func zipped(t *testing.T, s string) []byte {
+	var buf bytes.Buffer
+	w := zip.NewWriter(&buf)
+	f, _ := w.Create("report.xml")
+	f.Write([]byte(s))
+	w.Close()
+	return buf.Bytes()
+}
+
+func email(contentType, encoding string, body []byte) []byte {
+	b64 := base64.StdEncoding.EncodeToString(body)
+	var wrapped strings.Builder
+	for len(b64) > 76 {
+		wrapped.WriteString(b64[:76] + "\r\n")
+		b64 = b64[76:]
+	}
+	wrapped.WriteString(b64 + "\r\n")
+	return []byte("From: noreply@google.com\r\nSubject: Report Domain: example.com\r\nMIME-Version: 1.0\r\n" +
+		"Content-Type: multipart/mixed; boundary=\"XX\"\r\n\r\n--XX\r\nContent-Type: text/plain\r\n\r\nSee attached.\r\n" +
+		"--XX\r\nContent-Type: " + contentType + "\r\nContent-Transfer-Encoding: " + encoding + "\r\n\r\n" + wrapped.String() + "--XX--\r\n")
+}
+
+func TestParseForms(t *testing.T) {
+	forms := map[string][]byte{
+		"xml":        []byte("\n" + sample),
+		"gzip":       gz(t, sample),
+		"zip":        zipped(t, sample),
+		"mail+gzip":  email("application/gzip", "base64", gz(t, sample)),
+		"mail+zip":   email("application/zip; name=r.zip", "base64", zipped(t, sample)),
+		"mail+xml":   email("text/xml", "base64", []byte(sample)),
+		"mail+octet": email("application/octet-stream", "base64", gz(t, sample)),
+	}
+	for name, data := range forms {
+		t.Run(name, func(t *testing.T) {
+			r, err := Parse(data)
+			if err != nil {
+				t.Fatal(err)
+			}
+			check(t, r)
+		})
+	}
+}
+
+func TestParseRejects(t *testing.T) {
+	for name, data := range map[string][]byte{
+		"empty":       nil,
+		"text":        []byte("hello, this is not a report"),
+		"other xml":   []byte("<html><body/></html>"),
+		"no id":       []byte(strings.Replace(sample, "<report_id>12345</report_id>", "", 1)),
+		"mail no att": []byte("Subject: x\r\n\r\njust text\r\n"),
+		"bad gzip":    {0x1f, 0x8b, 0, 0},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if r, err := Parse(data); err == nil {
+				t.Fatalf("expected an error, got %+v", r)
+			}
+		})
+	}
+}
+
+func TestParseBombIsRefused(t *testing.T) {
+	big := "<feedback>" + strings.Repeat("x", maxXML+10) + "</feedback>"
+	if _, err := Parse(gz(t, big)); err == nil || !strings.Contains(err.Error(), "too large") {
+		t.Fatalf("want too large, got %v", err)
+	}
+}
+
+func TestPctDefaultsAndClamps(t *testing.T) {
+	r, err := Parse([]byte(strings.Replace(sample, "<pct>100</pct>", "<pct>25</pct>", 1)))
+	if err != nil || r.Policy.Pct != 25 {
+		t.Fatalf("pct 25: %v %+v", err, r)
+	}
+	r, err = Parse([]byte(strings.Replace(sample, "<pct>100</pct>", "", 1)))
+	if err != nil || r.Policy.Pct != 100 {
+		t.Fatalf("missing pct: %v %+v", err, r)
+	}
+}

model/model.go

@@ -1,7 +1,11 @@
 // Package model holds the types shared by hammerhead, blacktip and bamboo.
 package model
 
-import "time"
+import (
+	"sort"
+	"strings"
+	"time"
+)
 
 const (
 	ProtoTCP   = "tcp"   // raw TCP on ip:port
@@ -19,6 +23,21 @@ func (m Mapping) LastPort() int {
 	return m.Port
 }
 
+// IsRedirect reports whether the mapping answers with a redirect instead of forwarding to a target.
+func (m Mapping) IsRedirect() bool { return m.RedirectTo != "" }
+
+// RedirectLocation is where a request for requestURI (path and query) is sent. With KeepPath the
+// request's path and query follow the destination, so example.com/a?b=1 goes to to/a?b=1.
+func (m Mapping) RedirectLocation(requestURI string) string {
+	if !m.RedirectKeepPath {
+		return m.RedirectTo
+	}
+	if requestURI == "" || requestURI[0] != '/' {
+		requestURI = "/" + requestURI
+	}
+	return strings.TrimRight(m.RedirectTo, "/") + requestURI
+}
+
 // HostBased reports whether mappings of this protocol are routed by hostname.
 func HostBased(proto string) bool {
 	return proto == ProtoHTTP || proto == ProtoHTTPS || proto == ProtoTLS
@@ -39,6 +58,13 @@ type Edge struct {
 	OrgIDs     []string  `json:"orgIds"`            // orgs allowed to use this edge; empty = all
 	IPs        []string  `json:"ips"`               // addresses the edge reported it can bind
 	LastSeen   time.Time `json:"lastSeen"`
+
+	// Version is the release the program reports (e.g. v0.1.5). AutoUpdate says whether it may update
+	// itself; only then will it act on an update request (UpdateID). UpdateMsg is the outcome of the last requested update.
+	Version    string `json:"version,omitempty"`
+	AutoUpdate bool   `json:"autoUpdate,omitempty"`
+	UpdateID   string `json:"updateId,omitempty"` // set while an update requested by an administrator is pending
+	UpdateMsg  string `json:"updateMsg,omitempty"`
 }
 
 // Site is a bamboo agent: a private network that holds the real services.
@@ -54,8 +80,17 @@ type Site struct {
 	Local  bool   `json:"local"`
 	EdgeID string `json:"edgeId,omitempty"` // the edge a local site belongs to
 
+	Tags []string `json:"tags,omitempty"` // labels for grouping and filtering in the dashboard
+
 	TunnelIP  string `json:"tunnelIp,omitempty"`  // the site's address inside the tunnel network
 	PublicKey string `json:"publicKey,omitempty"` // WireGuard public key, reported by the agent
+
+	// Version is the release the program reports (e.g. v0.1.5). AutoUpdate says whether it may update
+	// itself; only then will it act on an update request (UpdateID). UpdateMsg is the outcome of the last requested update.
+	Version    string `json:"version,omitempty"`
+	AutoUpdate bool   `json:"autoUpdate,omitempty"`
+	UpdateID   string `json:"updateId,omitempty"` // set while an update requested by an administrator is pending
+	UpdateMsg  string `json:"updateMsg,omitempty"`
 }
 
 // Domain is a base domain an org owns. Hostnames of web/tls mappings must sit under one.
@@ -111,10 +146,17 @@ type Mapping struct {
 	Hostname string   `json:"hostname,omitempty"` // web/tls only; "*.example.com" allowed
 	Targets  []Target `json:"targets"`            // udp uses only the first
 	Enabled  bool     `json:"enabled"`
+	Tags     []string `json:"tags,omitempty"` // labels for grouping and filtering in the dashboard
 	// ProxyProtocol ("v1" or "v2") makes the edge start each tcp/tls connection to the
 	// target with a PROXY protocol header carrying the real client address.
 	ProxyProtocol string `json:"proxyProtocol,omitempty"`
 
+	// A web mapping with RedirectTo set sends visitors there instead of to a target (so it has no
+	// targets, login or health checks). It answers 301 when RedirectPermanent, otherwise 302.
+	RedirectTo        string `json:"redirectTo,omitempty"`
+	RedirectPermanent bool   `json:"redirectPermanent,omitempty"`
+	RedirectKeepPath  bool   `json:"redirectKeepPath,omitempty"`
+
 	// web (http/https) options
 	Auth          Auth   `json:"auth"`
 	AuthEpoch     int64  `json:"authEpoch"` // bumped to sign every visitor out; set by hammerhead only
@@ -200,6 +242,58 @@ type MailCheck struct {
 	HeloResolves   bool              `json:"heloResolves"`   // the HELO name resolves back to the source IP
 	Port25         string            `json:"port25"`         // "open" or why not
 	DKIM           map[string]string `json:"dkim"`           // key ID -> "published", "missing" or "mismatch"
+
+	Domains    map[string]DomainCheck `json:"domains,omitempty"`    // sending domain -> its DNS checks
+	Blocklists []string               `json:"blocklists,omitempty"` // DNS blocklists that list the source IP
+	Listable   int                    `json:"listable,omitempty"`   // how many blocklists answered, so "none listed" means something
+	Outcomes   *MailOutcomes          `json:"outcomes,omitempty"`   // what happened to the mail this egress sent
+}
+
+// Finding statuses.
+const (
+	FindingPass = "pass"
+	FindingWarn = "warn"
+	FindingFail = "fail"
+	FindingNone = "none" // nothing published
+)
+
+// MailFinding is the result of one deliverability check, with what to do about it.
+type MailFinding struct {
+	Status string `json:"status"`
+	Detail string `json:"detail"`
+	Hint   string `json:"hint,omitempty"`
+}
+
+// DomainCheck is how the DNS of one sending domain looks to a receiving server.
+type DomainCheck struct {
+	SPF    MailFinding `json:"spf"`
+	DMARC  MailFinding `json:"dmarc"`
+	MTASTS MailFinding `json:"mtaSts"`
+}
+
+// MailOutcomes counts what became of the mail an egress handled over a window.
+type MailOutcomes struct {
+	Since     time.Time               `json:"since"`
+	Delivered int                     `json:"delivered"`
+	Deferred  int                     `json:"deferred"`           // 4xx: the receiver asked us to try later
+	Failed    int                     `json:"failed"`             // 5xx: refused for good
+	ByDomain  map[string]OutcomeCount `json:"byDomain,omitempty"` // recipient domain
+	Recent    []OutcomeFailure        `json:"recent,omitempty"`   // latest problems, newest first
+}
+
+// OutcomeCount is delivered/deferred/failed for one recipient domain.
+type OutcomeCount struct {
+	Delivered int `json:"delivered"`
+	Deferred  int `json:"deferred"`
+	Failed    int `json:"failed"`
+}
+
+// OutcomeFailure is one deferral or refusal, for the dashboard to explain.
+type OutcomeFailure struct {
+	Time   time.Time `json:"time"`
+	Domain string    `json:"domain"`
+	Code   int       `json:"code"`
+	Text   string    `json:"text"`
 }
 
 // ---------- edge <-> hammerhead ----------
@@ -216,6 +310,7 @@ type EdgeConfig struct {
 	Local      []string  `json:"local"`    // site IDs the edge dials directly on its own machine
 	AcmeEmail  string    `json:"acmeEmail"`
 	AcmeStages bool      `json:"acmeStaging"`
+	UpdateID   string    `json:"updateId,omitempty"` // an administrator asked this edge to update itself now; echoed back in its report
 
 	MailEgress []MailPolicy `json:"mailEgress"`
 }
@@ -245,6 +340,42 @@ type Stat struct {
 	Conns    int64 `json:"conns"`
 	BytesIn  int64 `json:"bytesIn"`  // client -> target
 	BytesOut int64 `json:"bytesOut"` // target -> client
+	// Sources are the busiest client addresses behind these numbers (at most MaxSources),
+	// busiest first. Addresses beyond that are only counted in the totals above.
+	Sources []SourceStat `json:"sources,omitempty"`
+}
+
+// MaxSources is how many client addresses an edge names per port in one report.
+const MaxSources = 10
+
+// SourceStat is one client address's traffic on a port since the last report.
+type SourceStat struct {
+	IP       string `json:"ip"`
+	Conns    int64  `json:"conns"`
+	BytesIn  int64  `json:"bytesIn"`
+	BytesOut int64  `json:"bytesOut"`
+}
+
+// TopSources keeps the n busiest of a port's client addresses (by bytes, then connections), busiest first.
+func TopSources(all map[string]*SourceStat, n int) []SourceStat {
+	out := make([]SourceStat, 0, len(all))
+	for _, s := range all {
+		out = append(out, *s)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		a, b := out[i], out[j]
+		if ta, tb := a.BytesIn+a.BytesOut, b.BytesIn+b.BytesOut; ta != tb {
+			return ta > tb
+		}
+		if a.Conns != b.Conns {
+			return a.Conns > b.Conns
+		}
+		return a.IP < b.IP
+	})
+	if len(out) > n {
+		out = out[:n]
+	}
+	return out
 }
 
 // LogEntry is one access-log line.
@@ -281,14 +412,37 @@ type ContainerPort struct {
 	Type    string `json:"type"`
 }
 
+// DiscoveredService is a listening service an agent found on its own network (local sockets or an
+// opt-in scan of configured subnets). Hammerhead offers these as suggested targets; nothing is
+// proxied until an administrator picks one.
+type DiscoveredService struct {
+	Host    string `json:"host"` // address to dial, as seen from the agent
+	Port    int    `json:"port"`
+	Proto   string `json:"proto"`             // "tcp"
+	Service string `json:"service,omitempty"` // best guess: http, https, ssh, smtp, ftp, mysql, ... (empty = unknown)
+	Detail  string `json:"detail,omitempty"`  // banner or Server header, trimmed
+	Source  string `json:"source"`            // "local" (listening socket on the agent's host) or "scan"
+}
+
 // EdgeReport is what blacktip posts back.
 type EdgeReport struct {
-	IPs        []string                 `json:"ips"`
+	IPs       []string                 `json:"ips"`
-	PublicKey  string                   `json:"publicKey"` // the edge's WireGuard public key
+	PublicKey string                   `json:"publicKey"` // the edge's WireGuard public key
-	Status     map[string]MappingStatus `json:"status"`
+	Status    map[string]MappingStatus `json:"status"`
-	Stats      map[string]Stat          `json:"stats"` // deltas since last report
+	Stats     map[string]Stat          `json:"stats"` // deltas since last report
-	Containers []Container              `json:"containers"`
+	// StatsSeconds is how long Stats covers (the time since the previous report that carried them),
-	MailChecks map[string]MailCheck     `json:"mailChecks"` // by mail-egress ID
+	// so a rate can be worked out however irregular the reports are.
+	StatsSeconds float64              `json:"statsSeconds,omitempty"`
+	Containers   []Container          `json:"containers"`
+	MailChecks   map[string]MailCheck `json:"mailChecks"` // by mail-egress ID
+	Version      string               `json:"version,omitempty"`
+	AutoUpdate   bool                 `json:"autoUpdate,omitempty"`
+	UpdateMsg    string               `json:"updateMsg,omitempty"` // outcome of the update asked for by UpdateID
+	UpdateID     string               `json:"updateId,omitempty"`
+	// progress of that update, for the dashboard's progress bar and log
+	UpdateStage string   `json:"updateStage,omitempty"`
+	UpdatePct   int      `json:"updatePct,omitempty"`
+	UpdateLog   []string `json:"updateLog,omitempty"`
 }
 
 // EdgeRef tells a site where to dial.
@@ -306,16 +460,26 @@ type SiteConfig struct {
 	SiteID   string    `json:"siteId"`
 	TunnelIP string    `json:"tunnelIp"` // this site's WireGuard address
 	Edges    []EdgeRef `json:"edges"`
+	UpdateID string    `json:"updateId,omitempty"` // an administrator asked this agent to update itself now; echoed back in its report
 
 	MailEgress []MailSpec `json:"mailEgress"`
 }
 
 // SiteReport is what bamboo posts back.
 type SiteReport struct {
-	PublicKey  string            `json:"publicKey"`  // the agent's WireGuard public key
+	PublicKey  string              `json:"publicKey"`  // the agent's WireGuard public key
-	MailEgress map[string]string `json:"mailEgress"` // egress ID -> "" (listening) or an error
+	MailEgress map[string]string   `json:"mailEgress"` // egress ID -> "" (listening) or an error
-	Connected  []string          `json:"connected"`
+	Connected  []string            `json:"connected"`
-	Containers []Container       `json:"containers"`
+	Containers []Container         `json:"containers"`
+	Discovered []DiscoveredService `json:"discovered,omitempty"`
+	Version    string              `json:"version,omitempty"`
+	AutoUpdate bool                `json:"autoUpdate,omitempty"`
+	UpdateMsg  string              `json:"updateMsg,omitempty"`
+	UpdateID   string              `json:"updateId,omitempty"`
+	// progress of that update, for the dashboard's progress bar and log
+	UpdateStage string   `json:"updateStage,omitempty"`
+	UpdatePct   int      `json:"updatePct,omitempty"`
+	UpdateLog   []string `json:"updateLog,omitempty"`
 }
 
 // HostMatches reports whether host matches an exact name or a "*.example.com" pattern.

model/model_test.go

@@ -0,0 +1,41 @@
+package model
+
+import "testing"
+
+func TestRedirectLocation(t *testing.T) {
+	cases := []struct {
+		m    Mapping
+		uri  string
+		want string
+	}{
+		{Mapping{RedirectTo: "https://new.example.com"}, "/a?b=1", "https://new.example.com"},
+		{Mapping{RedirectTo: "https://new.example.com", RedirectKeepPath: true}, "/a?b=1", "https://new.example.com/a?b=1"},
+		{Mapping{RedirectTo: "https://new.example.com/", RedirectKeepPath: true}, "/", "https://new.example.com/"},
+		{Mapping{RedirectTo: "https://new.example.com/base", RedirectKeepPath: true}, "/x/y", "https://new.example.com/base/x/y"},
+		{Mapping{RedirectTo: "https://new.example.com", RedirectKeepPath: true}, "", "https://new.example.com/"},
+	}
+	for _, c := range cases {
+		if got := c.m.RedirectLocation(c.uri); got != c.want {
+			t.Errorf("%+v %q: got %q, want %q", c.m, c.uri, got, c.want)
+		}
+	}
+	if (Mapping{}).IsRedirect() || !(Mapping{RedirectTo: "https://x"}).IsRedirect() {
+		t.Error("IsRedirect is wrong")
+	}
+}
+
+func TestTopSources(t *testing.T) {
+	all := map[string]*SourceStat{
+		"a": {IP: "a", Conns: 1, BytesIn: 10, BytesOut: 5},
+		"b": {IP: "b", Conns: 9, BytesIn: 1},
+		"c": {IP: "c", Conns: 2, BytesIn: 100},
+		"d": {IP: "d", Conns: 3, BytesIn: 1},
+	}
+	got := TopSources(all, 3)
+	if len(got) != 3 || got[0].IP != "c" || got[1].IP != "a" || got[2].IP != "b" {
+		t.Fatalf("order: %+v", got) // c most bytes, a next, then b beats d on connections
+	}
+	if got := TopSources(all, 10); len(got) != 4 {
+		t.Fatalf("all: %+v", got)
+	}
+}

secretbox/secretbox.go

@@ -0,0 +1,87 @@
+// Package secretbox encrypts small secrets (tokens, passwords, private keys)
+// before they are written to the database, using AES-256-GCM.
+//
+// Sealed values look like "enc1:<base64>". Opening a value that isn't sealed
+// returns it unchanged, so data written before encryption was turned on is
+// still readable and gets sealed the next time it is saved.
+package secretbox
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"strings"
+)
+
+const prefix = "enc1:"
+
+type Box struct{ aead cipher.AEAD }
+
+// GenerateKey returns a new random key in the form New accepts.
+func GenerateKey() string {
+	b := make([]byte, 32)
+	rand.Read(b)
+	return base64.StdEncoding.EncodeToString(b)
+}
+
+// New builds a Box from a key given as base64 or hex of 32 bytes. Any other
+// text is treated as a passphrase and stretched with SHA-256.
+func New(key string) (*Box, error) {
+	key = strings.TrimSpace(key)
+	if key == "" {
+		return nil, errors.New("empty key")
+	}
+	raw := decodeKey(key)
+	block, err := aes.NewCipher(raw)
+	if err != nil {
+		return nil, err
+	}
+	aead, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	return &Box{aead: aead}, nil
+}
+
+func decodeKey(key string) []byte {
+	if b, err := base64.StdEncoding.DecodeString(key); err == nil && len(b) == 32 {
+		return b
+	}
+	if b, err := hex.DecodeString(key); err == nil && len(b) == 32 {
+		return b
+	}
+	sum := sha256.Sum256([]byte(key))
+	return sum[:]
+}
+
+// Seal encrypts s. The empty string stays empty so unset secrets stay visibly unset.
+func (b *Box) Seal(s string) string {
+	if s == "" || strings.HasPrefix(s, prefix) {
+		return s
+	}
+	nonce := make([]byte, b.aead.NonceSize())
+	rand.Read(nonce)
+	return prefix + base64.StdEncoding.EncodeToString(b.aead.Seal(nonce, nonce, []byte(s), nil))
+}
+
+// Open decrypts a sealed value. A wrong key or damaged data is an error, so a
+// bad key can never silently turn secrets into garbage.
+func (b *Box) Open(s string) (string, error) {
+	if !strings.HasPrefix(s, prefix) {
+		return s, nil
+	}
+	raw, err := base64.StdEncoding.DecodeString(s[len(prefix):])
+	if err != nil || len(raw) < b.aead.NonceSize() {
+		return "", errors.New("damaged secret")
+	}
+	n := b.aead.NonceSize()
+	plain, err := b.aead.Open(nil, raw[:n], raw[n:], nil)
+	if err != nil {
+		return "", errors.New("can't decrypt a stored secret: is HAMMERHEAD_MASTER_KEY the key it was saved with?")
+	}
+	return string(plain), nil
+}

secretbox/secretbox_test.go

@@ -0,0 +1,51 @@
+package secretbox
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestSealOpenRoundTripAndKeys(t *testing.T) {
+	key := GenerateKey()
+	b, err := New(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sealed := b.Seal("hunter2")
+	if !strings.HasPrefix(sealed, "enc1:") || strings.Contains(sealed, "hunter2") {
+		t.Fatalf("not sealed: %q", sealed)
+	}
+	if again := b.Seal("hunter2"); again == sealed {
+		t.Error("two seals of the same value must differ (random nonce)")
+	}
+	if b.Seal(sealed) != sealed {
+		t.Error("sealing an already sealed value must not double-encrypt")
+	}
+	if got, err := b.Open(sealed); err != nil || got != "hunter2" {
+		t.Errorf("Open = %q, %v", got, err)
+	}
+	if got, _ := b.Open("legacy plaintext"); got != "legacy plaintext" {
+		t.Errorf("unsealed legacy data should pass through, got %q", got)
+	}
+	if b.Seal("") != "" {
+		t.Error("empty stays empty")
+	}
+
+	other, _ := New(GenerateKey())
+	if _, err := other.Open(sealed); err == nil {
+		t.Error("the wrong key must fail loudly, not return garbage")
+	}
+	if _, err := b.Open(sealed[:len(sealed)-4] + "AAAA"); err == nil {
+		t.Error("tampered data must fail")
+	}
+
+	// a passphrase works as a key, and is stable
+	p1, _ := New("correct horse battery staple")
+	p2, _ := New("correct horse battery staple")
+	if got, err := p2.Open(p1.Seal("x")); err != nil || got != "x" {
+		t.Errorf("passphrase keys must be reproducible: %q %v", got, err)
+	}
+	if _, err := New("  "); err == nil {
+		t.Error("an empty key must be rejected")
+	}
+}

selfupdate/restart_unix.go

@@ -0,0 +1,13 @@
+//go:build !windows
+
+package selfupdate
+
+import (
+	"os"
+	"syscall"
+)
+
+// Restart replaces this process with the new binary, with the same arguments and environment.
+func Restart(exe string) error {
+	return syscall.Exec(exe, os.Args, os.Environ())
+}

selfupdate/restart_windows.go

@@ -0,0 +1,7 @@
+package selfupdate
+
+import "errors"
+
+func Restart(string) error {
+	return errors.New("restarting in place is not supported on Windows")
+}

selfupdate/selfupdate.go

@@ -0,0 +1,388 @@
+// Package selfupdate lets a program replace itself with the newest release published on a
+// Gitea server: it checks the latest release, downloads the archive for this platform,
+// verifies it against the release's SHA256SUMS, checks that the new binary runs, swaps it
+// in and restarts into it.
+package selfupdate
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const maxArchive = 300 << 20
+
+// Event is one step of an update, for showing progress: the stage ("checking", "downloading",
+// "verifying", "installing", "restarting", "done" or "failed"), how far along it is (0-100, or -1 to
+// leave the bar where it was) and, when there is something worth showing in a log, a line of text.
+type Event struct {
+	Stage string
+	Pct   int
+	Line  string
+}
+
+// Config says what to update and where from.
+type Config struct {
+	Server  string // e.g. https://git.veillax.com
+	Repo    string // owner/name, e.g. Veillax/blacktip
+	App     string // binary and archive prefix, e.g. blacktip
+	Current string // the running version, e.g. v0.1.2; anything that isn't vX.Y.Z is never updated
+	// Interval between checks (default 6 hours).
+	Interval time.Duration
+	Logf     func(format string, args ...any)
+	// Progress, if set, is told about every step of an update (it is called from the updating goroutine).
+	Progress func(Event)
+
+	// The fields below exist for tests and unusual setups.
+	HTTP    *http.Client
+	GOOS    string
+	GOARCH  string
+	ExePath string // binary to replace (default: the running one)
+	// Verify runs on the downloaded binary before it replaces the old one. The default
+	// runs it with -version and checks that it reports the new tag.
+	Verify func(path, tag string) error
+	// Restart is called after the swap (default: re-exec the new binary in place, keeping the
+	// process ID and environment).
+	Restart func(exe string) error
+}
+
+func (c *Config) defaults() {
+	if c.Interval <= 0 {
+		c.Interval = 6 * time.Hour
+	}
+	if c.Logf == nil {
+		c.Logf = func(string, ...any) {}
+	}
+	if c.HTTP == nil {
+		c.HTTP = &http.Client{Timeout: 5 * time.Minute}
+	}
+	if c.GOOS == "" {
+		c.GOOS = runtime.GOOS
+	}
+	if c.GOARCH == "" {
+		c.GOARCH = runtime.GOARCH
+	}
+	if c.Verify == nil {
+		c.Verify = verifyRuns(c.App)
+	}
+	if c.Restart == nil {
+		c.Restart = Restart
+	}
+	c.Server = strings.TrimRight(c.Server, "/")
+}
+
+// Run checks shortly after start and then every Interval until ctx ends. When an update
+// is installed, Restart takes over and Run normally does not return.
+func Run(ctx context.Context, cfg Config) {
+	cfg.defaults()
+	if _, ok := parseVersion(cfg.Current); !ok {
+		cfg.Logf("auto-update: this is a development build (%q), so it will not update itself", cfg.Current)
+		return
+	}
+	if cfg.GOOS == "windows" {
+		cfg.Logf("auto-update: not supported on Windows, because a running program can't be replaced; update it by hand")
+		return
+	}
+	wait := time.Duration(30+rand.Intn(240)) * time.Second // spread a fleet's first checks out
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(wait):
+		}
+		if _, err := CheckOnce(ctx, cfg); err != nil {
+			cfg.Logf("auto-update: %v", err)
+		}
+		wait = cfg.Interval + time.Duration(rand.Int63n(int64(cfg.Interval/10)+1))
+	}
+}
+
+func (c Config) say(stage string, pct int, line string) {
+	if c.Progress != nil {
+		c.Progress(Event{Stage: stage, Pct: pct, Line: line})
+	}
+}
+
+// CheckOnce looks for a newer release and installs it. It returns the new tag when it
+// installed one (and Restart returned, which only happens in tests or on failure).
+func CheckOnce(ctx context.Context, cfg Config) (string, error) {
+	cfg.defaults()
+	tag, err := checkOnce(ctx, cfg)
+	if err != nil {
+		cfg.say("failed", -1, "Failed: "+err.Error())
+	}
+	return tag, err
+}
+
+func checkOnce(ctx context.Context, cfg Config) (string, error) {
+	cur, ok := parseVersion(cfg.Current)
+	if !ok {
+		return "", fmt.Errorf("%q is not a release version", cfg.Current)
+	}
+	cfg.say("checking", 2, "Checking "+cfg.Server+" for the newest release of "+cfg.App)
+	rel, err := latestRelease(ctx, cfg)
+	if err != nil {
+		return "", err
+	}
+	if rel.Prerelease || rel.Draft {
+		cfg.say("done", 100, "Already on the newest release ("+cfg.Current+")")
+		return "", nil
+	}
+	latest, ok := parseVersion(rel.Tag)
+	if !ok || !latest.newerThan(cur) {
+		cfg.say("done", 100, "Already on the newest release ("+cfg.Current+")")
+		return "", nil // up to date, or something we don't understand: never downgrade
+	}
+
+	if cfg.GOOS == "windows" {
+		return "", errors.New("auto-update is not supported on Windows")
+	}
+	archive := fmt.Sprintf("%s_%s_%s_%s.tar.gz", cfg.App, rel.Tag, cfg.GOOS, cfg.GOARCH)
+	var archiveURL, sumsURL string
+	for _, a := range rel.Assets {
+		switch a.Name {
+		case archive:
+			archiveURL = a.URL
+		case "SHA256SUMS":
+			sumsURL = a.URL
+		}
+	}
+	if archiveURL == "" || sumsURL == "" {
+		return "", fmt.Errorf("release %s has no %s with SHA256SUMS yet (it may still be building); will retry", rel.Tag, archive)
+	}
+	cfg.Logf("auto-update: %s is available (running %s); downloading %s", rel.Tag, cfg.Current, archive)
+	cfg.say("checking", 5, rel.Tag+" is available (running "+cfg.Current+")")
+
+	sums, err := get(ctx, cfg.HTTP, sumsURL, 1<<20, nil)
+	if err != nil {
+		return "", err
+	}
+	want := ""
+	for _, line := range strings.Split(string(sums), "\n") {
+		f := strings.Fields(line)
+		if len(f) == 2 && strings.TrimPrefix(f[1], "*") == archive {
+			want = strings.ToLower(f[0])
+		}
+	}
+	if want == "" {
+		return "", fmt.Errorf("SHA256SUMS does not list %s", archive)
+	}
+	cfg.say("downloading", 6, "Downloading "+archive)
+	lastQuarter := 0
+	data, err := get(ctx, cfg.HTTP, archiveURL, maxArchive, func(done, total int64) {
+		if total <= 0 {
+			cfg.say("downloading", -1, "")
+			return
+		}
+		pct := 6 + int(80*done/total)
+		line := ""
+		if q := int(100 * done / total / 25); q > lastQuarter {
+			lastQuarter = q
+			line = fmt.Sprintf("Downloaded %.1f of %.1f MB (%d%%)", float64(done)/1e6, float64(total)/1e6, 100*done/total)
+		}
+		cfg.say("downloading", pct, line)
+	})
+	if err != nil {
+		return "", err
+	}
+	cfg.say("verifying", 88, "Verifying the checksum")
+	got := sha256.Sum256(data)
+	if hex.EncodeToString(got[:]) != want {
+		return "", fmt.Errorf("checksum mismatch for %s: not installing", archive)
+	}
+	cfg.say("verifying", 90, "The checksum matches")
+	cfg.say("installing", 92, "Unpacking the new binary")
+	bin, err := extract(data, cfg.App)
+	if err != nil {
+		return "", err
+	}
+
+	exe := cfg.ExePath
+	if exe == "" {
+		if exe, err = os.Executable(); err != nil {
+			return "", err
+		}
+		if exe, err = filepath.EvalSymlinks(exe); err != nil {
+			return "", err
+		}
+	}
+	tmp := filepath.Join(filepath.Dir(exe), "."+filepath.Base(exe)+".new")
+	if err := os.WriteFile(tmp, bin, 0o755); err != nil {
+		return "", fmt.Errorf("can't write next to %s (is its folder writable by this user?): %w", exe, err)
+	}
+	cfg.say("installing", 94, "Checking that the new binary starts")
+	if err := cfg.Verify(tmp, rel.Tag); err != nil {
+		os.Remove(tmp)
+		return "", fmt.Errorf("the new binary failed its check, keeping the old one: %w", err)
+	}
+	cfg.say("installing", 97, "Replacing "+exe)
+	if err := os.Rename(tmp, exe); err != nil { // atomic; the running process keeps its old file
+		os.Remove(tmp)
+		return "", fmt.Errorf("replacing %s: %w", exe, err)
+	}
+	cfg.Logf("auto-update: installed %s, restarting", rel.Tag)
+	cfg.say("restarting", 100, "Installed "+rel.Tag+"; restarting into it")
+	if err := cfg.Restart(exe); err != nil {
+		return rel.Tag, fmt.Errorf("installed %s but could not restart into it (it will start with the next restart): %w", rel.Tag, err)
+	}
+	return rel.Tag, nil
+}
+
+type release struct {
+	Tag        string `json:"tag_name"`
+	Prerelease bool   `json:"prerelease"`
+	Draft      bool   `json:"draft"`
+	Assets     []struct {
+		Name string `json:"name"`
+		URL  string `json:"browser_download_url"`
+	} `json:"assets"`
+}
+
+func latestRelease(ctx context.Context, cfg Config) (release, error) {
+	var rel release
+	body, err := get(ctx, cfg.HTTP, cfg.Server+"/api/v1/repos/"+cfg.Repo+"/releases/latest", 1<<20, nil)
+	if err != nil {
+		return rel, fmt.Errorf("checking for a new release: %w", err)
+	}
+	if err := json.Unmarshal(body, &rel); err != nil {
+		return rel, fmt.Errorf("reading the latest release: %w", err)
+	}
+	return rel, nil
+}
+
+// Check reports the newest full release and whether it is newer than cfg.Current, without
+// installing anything. A development build (a Current that isn't vX.Y.Z) is never "newer".
+func Check(ctx context.Context, cfg Config) (latest string, newer bool, err error) {
+	cfg.defaults()
+	rel, err := latestRelease(ctx, cfg)
+	if err != nil {
+		return "", false, err
+	}
+	if rel.Prerelease || rel.Draft {
+		return "", false, nil
+	}
+	l, ok := parseVersion(rel.Tag)
+	if !ok {
+		return rel.Tag, false, nil
+	}
+	cur, ok := parseVersion(cfg.Current)
+	return rel.Tag, ok && l.newerThan(cur), nil
+}
+
+func get(ctx context.Context, c *http.Client, url string, max int64, onProgress func(done, total int64)) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", "sharks-selfupdate")
+	resp, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("%s: %s", url, resp.Status)
+	}
+	var b []byte
+	buf := make([]byte, 64<<10)
+	for {
+		n, err := resp.Body.Read(buf)
+		b = append(b, buf[:n]...)
+		if int64(len(b)) > max {
+			return nil, fmt.Errorf("%s is larger than %d bytes", url, max)
+		}
+		if onProgress != nil && n > 0 {
+			onProgress(int64(len(b)), resp.ContentLength)
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	return b, nil
+}
+
+// extract finds the program inside a release archive (<pkg>/<app>).
+func extract(data []byte, app string) ([]byte, error) {
+	zr, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+	tr := tar.NewReader(zr)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return nil, fmt.Errorf("no %s inside the archive", app)
+		}
+		if err != nil {
+			return nil, err
+		}
+		if h.Typeflag == tar.TypeReg && path.Base(h.Name) == app && strings.Count(h.Name, "/") == 1 {
+			return io.ReadAll(io.LimitReader(tr, maxArchive))
+		}
+	}
+}
+
+func verifyRuns(app string) func(path, tag string) error {
+	return func(p, tag string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		out, err := exec.CommandContext(ctx, p, "-version").Output()
+		if err != nil {
+			return err
+		}
+		if want := app + " " + tag; strings.TrimSpace(string(out)) != want {
+			return fmt.Errorf("it reports %q, expected %q", strings.TrimSpace(string(out)), want)
+		}
+		return nil
+	}
+}
+
+type version [3]int
+
+func parseVersion(s string) (version, bool) {
+	var v version
+	if !strings.HasPrefix(s, "v") || strings.ContainsAny(s, "-+") {
+		return v, false // pre-releases and local builds are never part of auto-update
+	}
+	parts := strings.Split(s[1:], ".")
+	if len(parts) != 3 {
+		return v, false
+	}
+	for i, p := range parts {
+		n, err := strconv.Atoi(p)
+		if err != nil || n < 0 {
+			return v, false
+		}
+		v[i] = n
+	}
+	return v, true
+}
+
+func (a version) newerThan(b version) bool {
+	for i := range a {
+		if a[i] != b[i] {
+			return a[i] > b[i]
+		}
+	}
+	return false
+}

selfupdate/selfupdate_test.go

@@ -0,0 +1,197 @@
+package selfupdate
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func makeArchive(t *testing.T, pkg, app, content string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	gz := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gz)
+	for name, body := range map[string]string{pkg + "/README.md": "hi", pkg + "/" + app: content} {
+		if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o755, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil {
+			t.Fatal(err)
+		}
+		tw.Write([]byte(body))
+	}
+	tw.Close()
+	gz.Close()
+	return buf.Bytes()
+}
+
+// fakeGitea serves one release with the given tag.
+func fakeGitea(t *testing.T, tag string, prerelease bool, corrupt bool) *httptest.Server {
+	archive := fmt.Sprintf("demo_%s_linux_amd64.tar.gz", tag)
+	data := makeArchive(t, strings.TrimSuffix(archive, ".tar.gz"), "demo", "NEW-BINARY "+tag)
+	sum := sha256.Sum256(data)
+	sums := hex.EncodeToString(sum[:]) + "  " + archive + "\n"
+	if corrupt {
+		sums = strings.Repeat("0", 64) + "  " + archive + "\n"
+	}
+	var srv *httptest.Server
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/v1/repos/Veillax/demo/releases/latest", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, `{"tag_name":%q,"prerelease":%v,"assets":[{"name":%q,"browser_download_url":"%s/dl/%s"},{"name":"SHA256SUMS","browser_download_url":"%s/dl/SHA256SUMS"}]}`,
+			tag, prerelease, archive, srv.URL, archive, srv.URL)
+	})
+	mux.HandleFunc("/dl/SHA256SUMS", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, sums) })
+	mux.HandleFunc("/dl/"+archive, func(w http.ResponseWriter, r *http.Request) { w.Write(data) })
+	srv = httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func setup(t *testing.T, srv *httptest.Server, current string) (Config, string, *string) {
+	exe := filepath.Join(t.TempDir(), "demo")
+	os.WriteFile(exe, []byte("OLD-BINARY"), 0o755)
+	restarted := new(string)
+	return Config{Server: srv.URL, Repo: "Veillax/demo", App: "demo", Current: current, GOOS: "linux", GOARCH: "amd64", ExePath: exe,
+		Verify:  func(p, tag string) error { return nil },
+		Restart: func(e string) error { *restarted = e; return nil }}, exe, restarted
+}
+
+func TestInstallsANewerRelease(t *testing.T) {
+	cfg, exe, restarted := setup(t, fakeGitea(t, "v0.2.0", false, false), "v0.1.5")
+	tag, err := CheckOnce(context.Background(), cfg)
+	if err != nil || tag != "v0.2.0" {
+		t.Fatalf("tag=%q err=%v", tag, err)
+	}
+	if b, _ := os.ReadFile(exe); string(b) != "NEW-BINARY v0.2.0" {
+		t.Fatalf("binary is %q", b)
+	}
+	if *restarted != exe {
+		t.Fatalf("restart called with %q", *restarted)
+	}
+	if _, err := os.Stat(filepath.Join(filepath.Dir(exe), ".demo.new")); err == nil {
+		t.Fatal("temporary file left behind")
+	}
+}
+
+func TestLeavesThingsAloneWhenItShould(t *testing.T) {
+	for _, c := range []struct {
+		name, current, tag string
+		pre                bool
+	}{
+		{"same version", "v0.2.0", "v0.2.0", false},
+		{"never downgrades", "v0.3.0", "v0.2.0", false},
+		{"pre-release is skipped", "v0.1.0", "v0.2.0", true},
+		{"dev build", "dev", "v0.2.0", false},
+		{"local build", "v0.2.0-local", "v0.9.0", false},
+	} {
+		cfg, exe, restarted := setup(t, fakeGitea(t, c.tag, c.pre, false), c.current)
+		tag, _ := CheckOnce(context.Background(), cfg)
+		if b, _ := os.ReadFile(exe); tag != "" || string(b) != "OLD-BINARY" || *restarted != "" {
+			t.Errorf("%s: changed things (tag=%q binary=%q)", c.name, tag, b)
+		}
+	}
+}
+
+func TestRefusesABadChecksum(t *testing.T) {
+	cfg, exe, restarted := setup(t, fakeGitea(t, "v0.2.0", false, true), "v0.1.0")
+	if _, err := CheckOnce(context.Background(), cfg); err == nil || !strings.Contains(err.Error(), "checksum") {
+		t.Fatalf("err=%v", err)
+	}
+	if b, _ := os.ReadFile(exe); string(b) != "OLD-BINARY" || *restarted != "" {
+		t.Fatal("installed a binary that failed its checksum")
+	}
+}
+
+func TestKeepsTheOldBinaryWhenTheNewOneFailsItsCheck(t *testing.T) {
+	cfg, exe, restarted := setup(t, fakeGitea(t, "v0.2.0", false, false), "v0.1.0")
+	cfg.Verify = func(p, tag string) error { return fmt.Errorf("segfault") }
+	if _, err := CheckOnce(context.Background(), cfg); err == nil {
+		t.Fatal("expected an error")
+	}
+	if b, _ := os.ReadFile(exe); string(b) != "OLD-BINARY" || *restarted != "" {
+		t.Fatal("replaced the binary despite a failed check")
+	}
+	if _, err := os.Stat(filepath.Join(filepath.Dir(exe), ".demo.new")); err == nil {
+		t.Fatal("temporary file left behind")
+	}
+}
+
+func TestVersionOrdering(t *testing.T) {
+	for _, c := range []struct {
+		a, b string
+		want bool
+	}{{"v0.1.10", "v0.1.9", true}, {"v1.0.0", "v0.99.99", true}, {"v0.1.2", "v0.1.2", false}, {"v0.1.1", "v0.2.0", false}} {
+		a, _ := parseVersion(c.a)
+		b, _ := parseVersion(c.b)
+		if a.newerThan(b) != c.want {
+			t.Errorf("%s newer than %s: want %v", c.a, c.b, c.want)
+		}
+	}
+}
+
+func TestCheckReportsWithoutInstalling(t *testing.T) {
+	srv := fakeGitea(t, "v0.2.0", false, false)
+	for _, c := range []struct {
+		current string
+		newer   bool
+	}{{"v0.1.0", true}, {"v0.2.0", false}, {"v0.3.0", false}, {"dev", false}, {"v0.1.0-local", false}} {
+		cfg, exe, restarted := setup(t, srv, c.current)
+		tag, newer, err := Check(context.Background(), cfg)
+		if err != nil || tag != "v0.2.0" || newer != c.newer {
+			t.Errorf("%s: tag=%q newer=%v err=%v", c.current, tag, newer, err)
+		}
+		if b, _ := os.ReadFile(exe); string(b) != "OLD-BINARY" || *restarted != "" {
+			t.Errorf("%s: Check changed things", c.current)
+		}
+	}
+}
+
+func TestReportsProgressThroughTheWholeUpdate(t *testing.T) {
+	cfg, _, _ := setup(t, fakeGitea(t, "v0.2.0", false, false), "v0.1.0")
+	var events []Event
+	cfg.Progress = func(e Event) { events = append(events, e) }
+	if _, err := CheckOnce(context.Background(), cfg); err != nil {
+		t.Fatal(err)
+	}
+	var stages []string
+	last, lines := 0, ""
+	for _, e := range events {
+		if len(stages) == 0 || stages[len(stages)-1] != e.Stage {
+			stages = append(stages, e.Stage)
+		}
+		if e.Pct >= 0 {
+			if e.Pct < last {
+				t.Errorf("progress went backwards: %d after %d (%+v)", e.Pct, last, e)
+			}
+			last = e.Pct
+		}
+		lines += e.Line + "\n"
+	}
+	if got := strings.Join(stages, ","); got != "checking,downloading,verifying,installing,restarting" {
+		t.Errorf("stages: %s", got)
+	}
+	if last != 100 || !strings.Contains(lines, "v0.2.0 is available") || !strings.Contains(lines, "checksum matches") {
+		t.Errorf("progress ended at %d with log:\n%s", last, lines)
+	}
+}
+
+func TestProgressReportsFailureAndNothingToDo(t *testing.T) {
+	cfg, _, _ := setup(t, fakeGitea(t, "v0.2.0", false, true), "v0.1.0")
+	var last Event
+	cfg.Progress = func(e Event) { last = e }
+	if _, err := CheckOnce(context.Background(), cfg); err == nil || last.Stage != "failed" || !strings.Contains(last.Line, "checksum") {
+		t.Errorf("err=%v last=%+v", err, last)
+	}
+	cfg, _, _ = setup(t, fakeGitea(t, "v0.2.0", false, false), "v0.2.0")
+	cfg.Progress = func(e Event) { last = e }
+	if tag, err := CheckOnce(context.Background(), cfg); err != nil || tag != "" || last.Stage != "done" || last.Pct != 100 {
+		t.Errorf("tag=%q err=%v last=%+v", tag, err, last)
+	}
+}