model: Site and Mapping carry optional tags

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

README.md

@@ -1,6 +1,6 @@
 # Reef
 
-A shared library for Hammerhead, Blacktip, and Bamboo
+A shared library for Hammerhead, Blacktip, Bamboo and Sandbar
 
 Shared types and building blocks for the sharks stack: [Hammerhead](https://git.veillax.com/Veillax/hammerhead) (control plane), [Blacktip](https://git.veillax.com/Veillax/blacktip) (edge) and [Bamboo](https://git.veillax.com/Veillax/bamboo) (site agent).
 
@@ -15,6 +15,7 @@ Reef holds the code all three programs agree on, so each can be built and releas
 | `netguard` | Refuses internal addresses for anything that connects out on a user's behalf (webhooks, mail delivery), and dials the address it checked |
 | `docker` | Lists running containers through the Docker Engine API, with no SDK |
 | `dotenv` | Loads a `.env` file into the environment without overriding real variables |
+| `secretbox` | AES-256-GCM sealing of small secrets (`enc1:` values) before they are written to a database |
 
 ## Using it
 

model/model.go

@@ -39,6 +39,13 @@ type Edge struct {
 	OrgIDs     []string  `json:"orgIds"`            // orgs allowed to use this edge; empty = all
 	IPs        []string  `json:"ips"`               // addresses the edge reported it can bind
 	LastSeen   time.Time `json:"lastSeen"`
+
+	// Version is the release the program reports (e.g. v0.1.5). AutoUpdate says whether it may update
+	// itself; only then will it act on an update request (UpdateID). UpdateMsg is the outcome of the last requested update.
+	Version    string `json:"version,omitempty"`
+	AutoUpdate bool   `json:"autoUpdate,omitempty"`
+	UpdateID   string `json:"updateId,omitempty"` // set while an update requested by an administrator is pending
+	UpdateMsg  string `json:"updateMsg,omitempty"`
 }
 
 // Site is a bamboo agent: a private network that holds the real services.
@@ -54,8 +61,17 @@ type Site struct {
 	Local  bool   `json:"local"`
 	EdgeID string `json:"edgeId,omitempty"` // the edge a local site belongs to
 
-	TunnelIP  string `json:"tunnelIp,omitempty"`  // the site's address inside the tunnel network
+	Tags []string `json:"tags,omitempty"` // labels for grouping and filtering in the dashboard
+
+	TunnelIP string `json:"tunnelIp,omitempty"`  // the site's address inside the tunnel network
 	PublicKey string `json:"publicKey,omitempty"` // WireGuard public key, reported by the agent
+
+	// Version is the release the program reports (e.g. v0.1.5). AutoUpdate says whether it may update
+	// itself; only then will it act on an update request (UpdateID). UpdateMsg is the outcome of the last requested update.
+	Version    string `json:"version,omitempty"`
+	AutoUpdate bool   `json:"autoUpdate,omitempty"`
+	UpdateID   string `json:"updateId,omitempty"` // set while an update requested by an administrator is pending
+	UpdateMsg  string `json:"updateMsg,omitempty"`
 }
 
 // Domain is a base domain an org owns. Hostnames of web/tls mappings must sit under one.
@@ -111,6 +127,7 @@ type Mapping struct {
 	Hostname string   `json:"hostname,omitempty"` // web/tls only; "*.example.com" allowed
 	Targets  []Target `json:"targets"`            // udp uses only the first
 	Enabled  bool     `json:"enabled"`
+	Tags     []string `json:"tags,omitempty"` // labels for grouping and filtering in the dashboard
 	// ProxyProtocol ("v1" or "v2") makes the edge start each tcp/tls connection to the
 	// target with a PROXY protocol header carrying the real client address.
 	ProxyProtocol string `json:"proxyProtocol,omitempty"`
@@ -216,6 +233,7 @@ type EdgeConfig struct {
 	Local      []string  `json:"local"`    // site IDs the edge dials directly on its own machine
 	AcmeEmail  string    `json:"acmeEmail"`
 	AcmeStages bool      `json:"acmeStaging"`
+	UpdateID   string    `json:"updateId,omitempty"` // an administrator asked this edge to update itself now; echoed back in its report
 
 	MailEgress []MailPolicy `json:"mailEgress"`
 }
@@ -289,6 +307,14 @@ type EdgeReport struct {
 	Stats      map[string]Stat          `json:"stats"` // deltas since last report
 	Containers []Container              `json:"containers"`
 	MailChecks map[string]MailCheck     `json:"mailChecks"` // by mail-egress ID
+	Version    string                   `json:"version,omitempty"`
+	AutoUpdate bool                     `json:"autoUpdate,omitempty"`
+	UpdateMsg  string                   `json:"updateMsg,omitempty"` // outcome of the update asked for by UpdateID
+	UpdateID   string                   `json:"updateId,omitempty"`
+	// progress of that update, for the dashboard's progress bar and log
+	UpdateStage string   `json:"updateStage,omitempty"`
+	UpdatePct   int      `json:"updatePct,omitempty"`
+	UpdateLog   []string `json:"updateLog,omitempty"`
 }
 
 // EdgeRef tells a site where to dial.
@@ -306,6 +332,7 @@ type SiteConfig struct {
 	SiteID   string    `json:"siteId"`
 	TunnelIP string    `json:"tunnelIp"` // this site's WireGuard address
 	Edges    []EdgeRef `json:"edges"`
+	UpdateID string    `json:"updateId,omitempty"` // an administrator asked this agent to update itself now; echoed back in its report
 
 	MailEgress []MailSpec `json:"mailEgress"`
 }
@@ -316,6 +343,14 @@ type SiteReport struct {
 	MailEgress map[string]string `json:"mailEgress"` // egress ID -> "" (listening) or an error
 	Connected  []string          `json:"connected"`
 	Containers []Container       `json:"containers"`
+	Version    string            `json:"version,omitempty"`
+	AutoUpdate bool              `json:"autoUpdate,omitempty"`
+	UpdateMsg  string            `json:"updateMsg,omitempty"`
+	UpdateID   string            `json:"updateId,omitempty"`
+	// progress of that update, for the dashboard's progress bar and log
+	UpdateStage string   `json:"updateStage,omitempty"`
+	UpdatePct   int      `json:"updatePct,omitempty"`
+	UpdateLog   []string `json:"updateLog,omitempty"`
 }
 
 // HostMatches reports whether host matches an exact name or a "*.example.com" pattern.

secretbox/secretbox.go

@@ -0,0 +1,87 @@
+// Package secretbox encrypts small secrets (tokens, passwords, private keys)
+// before they are written to the database, using AES-256-GCM.
+//
+// Sealed values look like "enc1:<base64>". Opening a value that isn't sealed
+// returns it unchanged, so data written before encryption was turned on is
+// still readable and gets sealed the next time it is saved.
+package secretbox
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"strings"
+)
+
+const prefix = "enc1:"
+
+type Box struct{ aead cipher.AEAD }
+
+// GenerateKey returns a new random key in the form New accepts.
+func GenerateKey() string {
+	b := make([]byte, 32)
+	rand.Read(b)
+	return base64.StdEncoding.EncodeToString(b)
+}
+
+// New builds a Box from a key given as base64 or hex of 32 bytes. Any other
+// text is treated as a passphrase and stretched with SHA-256.
+func New(key string) (*Box, error) {
+	key = strings.TrimSpace(key)
+	if key == "" {
+		return nil, errors.New("empty key")
+	}
+	raw := decodeKey(key)
+	block, err := aes.NewCipher(raw)
+	if err != nil {
+		return nil, err
+	}
+	aead, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	return &Box{aead: aead}, nil
+}
+
+func decodeKey(key string) []byte {
+	if b, err := base64.StdEncoding.DecodeString(key); err == nil && len(b) == 32 {
+		return b
+	}
+	if b, err := hex.DecodeString(key); err == nil && len(b) == 32 {
+		return b
+	}
+	sum := sha256.Sum256([]byte(key))
+	return sum[:]
+}
+
+// Seal encrypts s. The empty string stays empty so unset secrets stay visibly unset.
+func (b *Box) Seal(s string) string {
+	if s == "" || strings.HasPrefix(s, prefix) {
+		return s
+	}
+	nonce := make([]byte, b.aead.NonceSize())
+	rand.Read(nonce)
+	return prefix + base64.StdEncoding.EncodeToString(b.aead.Seal(nonce, nonce, []byte(s), nil))
+}
+
+// Open decrypts a sealed value. A wrong key or damaged data is an error, so a
+// bad key can never silently turn secrets into garbage.
+func (b *Box) Open(s string) (string, error) {
+	if !strings.HasPrefix(s, prefix) {
+		return s, nil
+	}
+	raw, err := base64.StdEncoding.DecodeString(s[len(prefix):])
+	if err != nil || len(raw) < b.aead.NonceSize() {
+		return "", errors.New("damaged secret")
+	}
+	n := b.aead.NonceSize()
+	plain, err := b.aead.Open(nil, raw[:n], raw[n:], nil)
+	if err != nil {
+		return "", errors.New("can't decrypt a stored secret: is HAMMERHEAD_MASTER_KEY the key it was saved with?")
+	}
+	return string(plain), nil
+}

secretbox/secretbox_test.go

@@ -0,0 +1,51 @@
+package secretbox
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestSealOpenRoundTripAndKeys(t *testing.T) {
+	key := GenerateKey()
+	b, err := New(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sealed := b.Seal("hunter2")
+	if !strings.HasPrefix(sealed, "enc1:") || strings.Contains(sealed, "hunter2") {
+		t.Fatalf("not sealed: %q", sealed)
+	}
+	if again := b.Seal("hunter2"); again == sealed {
+		t.Error("two seals of the same value must differ (random nonce)")
+	}
+	if b.Seal(sealed) != sealed {
+		t.Error("sealing an already sealed value must not double-encrypt")
+	}
+	if got, err := b.Open(sealed); err != nil || got != "hunter2" {
+		t.Errorf("Open = %q, %v", got, err)
+	}
+	if got, _ := b.Open("legacy plaintext"); got != "legacy plaintext" {
+		t.Errorf("unsealed legacy data should pass through, got %q", got)
+	}
+	if b.Seal("") != "" {
+		t.Error("empty stays empty")
+	}
+
+	other, _ := New(GenerateKey())
+	if _, err := other.Open(sealed); err == nil {
+		t.Error("the wrong key must fail loudly, not return garbage")
+	}
+	if _, err := b.Open(sealed[:len(sealed)-4] + "AAAA"); err == nil {
+		t.Error("tampered data must fail")
+	}
+
+	// a passphrase works as a key, and is stable
+	p1, _ := New("correct horse battery staple")
+	p2, _ := New("correct horse battery staple")
+	if got, err := p2.Open(p1.Seal("x")); err != nil || got != "x" {
+		t.Errorf("passphrase keys must be reproducible: %q %v", got, err)
+	}
+	if _, err := New("  "); err == nil {
+		t.Error("an empty key must be rejected")
+	}
+}

selfupdate/selfupdate.go

@@ -29,6 +29,15 @@ import (
 
 const maxArchive = 300 << 20
 
+// Event is one step of an update, for showing progress: the stage ("checking", "downloading",
+// "verifying", "installing", "restarting", "done" or "failed"), how far along it is (0-100, or -1 to
+// leave the bar where it was) and, when there is something worth showing in a log, a line of text.
+type Event struct {
+	Stage string
+	Pct   int
+	Line  string
+}
+
 // Config says what to update and where from.
 type Config struct {
 	Server  string // e.g. https://git.veillax.com
@@ -38,6 +47,8 @@ type Config struct {
 	// Interval between checks (default 6 hours).
 	Interval time.Duration
 	Logf     func(format string, args ...any)
+	// Progress, if set, is told about every step of an update (it is called from the updating goroutine).
+	Progress func(Event)
 
 	// The fields below exist for tests and unusual setups.
 	HTTP    *http.Client
@@ -103,23 +114,40 @@ func Run(ctx context.Context, cfg Config) {
 	}
 }
 
+func (c Config) say(stage string, pct int, line string) {
+	if c.Progress != nil {
+		c.Progress(Event{Stage: stage, Pct: pct, Line: line})
+	}
+}
+
 // CheckOnce looks for a newer release and installs it. It returns the new tag when it
 // installed one (and Restart returned, which only happens in tests or on failure).
 func CheckOnce(ctx context.Context, cfg Config) (string, error) {
 	cfg.defaults()
+	tag, err := checkOnce(ctx, cfg)
+	if err != nil {
+		cfg.say("failed", -1, "Failed: "+err.Error())
+	}
+	return tag, err
+}
+
+func checkOnce(ctx context.Context, cfg Config) (string, error) {
 	cur, ok := parseVersion(cfg.Current)
 	if !ok {
 		return "", fmt.Errorf("%q is not a release version", cfg.Current)
 	}
+	cfg.say("checking", 2, "Checking "+cfg.Server+" for the newest release of "+cfg.App)
 	rel, err := latestRelease(ctx, cfg)
 	if err != nil {
 		return "", err
 	}
 	if rel.Prerelease || rel.Draft {
+		cfg.say("done", 100, "Already on the newest release ("+cfg.Current+")")
 		return "", nil
 	}
 	latest, ok := parseVersion(rel.Tag)
 	if !ok || !latest.newerThan(cur) {
+		cfg.say("done", 100, "Already on the newest release ("+cfg.Current+")")
 		return "", nil // up to date, or something we don't understand: never downgrade
 	}
 
@@ -140,8 +168,9 @@ func CheckOnce(ctx context.Context, cfg Config) (string, error) {
 		return "", fmt.Errorf("release %s has no %s with SHA256SUMS yet (it may still be building); will retry", rel.Tag, archive)
 	}
 	cfg.Logf("auto-update: %s is available (running %s); downloading %s", rel.Tag, cfg.Current, archive)
+	cfg.say("checking", 5, rel.Tag+" is available (running "+cfg.Current+")")
 
-	sums, err := get(ctx, cfg.HTTP, sumsURL, 1<<20)
+	sums, err := get(ctx, cfg.HTTP, sumsURL, 1<<20, nil)
 	if err != nil {
 		return "", err
 	}
@@ -155,14 +184,31 @@ func CheckOnce(ctx context.Context, cfg Config) (string, error) {
 	if want == "" {
 		return "", fmt.Errorf("SHA256SUMS does not list %s", archive)
 	}
-	data, err := get(ctx, cfg.HTTP, archiveURL, maxArchive)
+	cfg.say("downloading", 6, "Downloading "+archive)
+	lastQuarter := 0
+	data, err := get(ctx, cfg.HTTP, archiveURL, maxArchive, func(done, total int64) {
+		if total <= 0 {
+			cfg.say("downloading", -1, "")
+			return
+		}
+		pct := 6 + int(80*done/total)
+		line := ""
+		if q := int(100 * done / total / 25); q > lastQuarter {
+			lastQuarter = q
+			line = fmt.Sprintf("Downloaded %.1f of %.1f MB (%d%%)", float64(done)/1e6, float64(total)/1e6, 100*done/total)
+		}
+		cfg.say("downloading", pct, line)
+	})
 	if err != nil {
 		return "", err
 	}
+	cfg.say("verifying", 88, "Verifying the checksum")
 	got := sha256.Sum256(data)
 	if hex.EncodeToString(got[:]) != want {
 		return "", fmt.Errorf("checksum mismatch for %s: not installing", archive)
 	}
+	cfg.say("verifying", 90, "The checksum matches")
+	cfg.say("installing", 92, "Unpacking the new binary")
 	bin, err := extract(data, cfg.App)
 	if err != nil {
 		return "", err
@@ -181,15 +227,18 @@ func CheckOnce(ctx context.Context, cfg Config) (string, error) {
 	if err := os.WriteFile(tmp, bin, 0o755); err != nil {
 		return "", fmt.Errorf("can't write next to %s (is its folder writable by this user?): %w", exe, err)
 	}
+	cfg.say("installing", 94, "Checking that the new binary starts")
 	if err := cfg.Verify(tmp, rel.Tag); err != nil {
 		os.Remove(tmp)
 		return "", fmt.Errorf("the new binary failed its check, keeping the old one: %w", err)
 	}
+	cfg.say("installing", 97, "Replacing "+exe)
 	if err := os.Rename(tmp, exe); err != nil { // atomic; the running process keeps its old file
 		os.Remove(tmp)
 		return "", fmt.Errorf("replacing %s: %w", exe, err)
 	}
 	cfg.Logf("auto-update: installed %s, restarting", rel.Tag)
+	cfg.say("restarting", 100, "Installed "+rel.Tag+"; restarting into it")
 	if err := cfg.Restart(exe); err != nil {
 		return rel.Tag, fmt.Errorf("installed %s but could not restart into it (it will start with the next restart): %w", rel.Tag, err)
 	}
@@ -208,7 +257,7 @@ type release struct {
 
 func latestRelease(ctx context.Context, cfg Config) (release, error) {
 	var rel release
-	body, err := get(ctx, cfg.HTTP, cfg.Server+"/api/v1/repos/"+cfg.Repo+"/releases/latest", 1<<20)
+	body, err := get(ctx, cfg.HTTP, cfg.Server+"/api/v1/repos/"+cfg.Repo+"/releases/latest", 1<<20, nil)
 	if err != nil {
 		return rel, fmt.Errorf("checking for a new release: %w", err)
 	}
@@ -237,7 +286,7 @@ func Check(ctx context.Context, cfg Config) (latest string, newer bool, err erro
 	return rel.Tag, ok && l.newerThan(cur), nil
 }
 
-func get(ctx context.Context, c *http.Client, url string, max int64) ([]byte, error) {
+func get(ctx context.Context, c *http.Client, url string, max int64, onProgress func(done, total int64)) ([]byte, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
 		return nil, err
@@ -251,12 +300,23 @@ func get(ctx context.Context, c *http.Client, url string, max int64) ([]byte, er
 	if resp.StatusCode != 200 {
 		return nil, fmt.Errorf("%s: %s", url, resp.Status)
 	}
-	b, err := io.ReadAll(io.LimitReader(resp.Body, max+1))
-	if err != nil {
-		return nil, err
-	}
-	if int64(len(b)) > max {
-		return nil, fmt.Errorf("%s is larger than %d bytes", url, max)
+	var b []byte
+	buf := make([]byte, 64<<10)
+	for {
+		n, err := resp.Body.Read(buf)
+		b = append(b, buf[:n]...)
+		if int64(len(b)) > max {
+			return nil, fmt.Errorf("%s is larger than %d bytes", url, max)
+		}
+		if onProgress != nil && n > 0 {
+			onProgress(int64(len(b)), resp.ContentLength)
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
 	}
 	return b, nil
 }

selfupdate/selfupdate_test.go

@@ -152,3 +152,46 @@ func TestCheckReportsWithoutInstalling(t *testing.T) {
 		}
 	}
 }
+
+func TestReportsProgressThroughTheWholeUpdate(t *testing.T) {
+	cfg, _, _ := setup(t, fakeGitea(t, "v0.2.0", false, false), "v0.1.0")
+	var events []Event
+	cfg.Progress = func(e Event) { events = append(events, e) }
+	if _, err := CheckOnce(context.Background(), cfg); err != nil {
+		t.Fatal(err)
+	}
+	var stages []string
+	last, lines := 0, ""
+	for _, e := range events {
+		if len(stages) == 0 || stages[len(stages)-1] != e.Stage {
+			stages = append(stages, e.Stage)
+		}
+		if e.Pct >= 0 {
+			if e.Pct < last {
+				t.Errorf("progress went backwards: %d after %d (%+v)", e.Pct, last, e)
+			}
+			last = e.Pct
+		}
+		lines += e.Line + "\n"
+	}
+	if got := strings.Join(stages, ","); got != "checking,downloading,verifying,installing,restarting" {
+		t.Errorf("stages: %s", got)
+	}
+	if last != 100 || !strings.Contains(lines, "v0.2.0 is available") || !strings.Contains(lines, "checksum matches") {
+		t.Errorf("progress ended at %d with log:\n%s", last, lines)
+	}
+}
+
+func TestProgressReportsFailureAndNothingToDo(t *testing.T) {
+	cfg, _, _ := setup(t, fakeGitea(t, "v0.2.0", false, true), "v0.1.0")
+	var last Event
+	cfg.Progress = func(e Event) { last = e }
+	if _, err := CheckOnce(context.Background(), cfg); err == nil || last.Stage != "failed" || !strings.Contains(last.Line, "checksum") {
+		t.Errorf("err=%v last=%+v", err, last)
+	}
+	cfg, _, _ = setup(t, fakeGitea(t, "v0.2.0", false, false), "v0.2.0")
+	cfg.Progress = func(e Event) { last = e }
+	if tag, err := CheckOnce(context.Background(), cfg); err != nil || tag != "" || last.Stage != "done" || last.Pct != 100 {
+		t.Errorf("tag=%q err=%v last=%+v", tag, err, last)
+	}
+}