Report update progress and log to Hammerhead while updating

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.gitea/workflows/ci.yml

@@ -44,7 +44,7 @@ jobs:
       # Catches platform-specific breakage before a release is tagged.
       - name: Every release platform compiles
         run: |
-          for t in linux/amd64 linux/arm64 linux/arm windows/amd64 darwin/amd64 darwin/arm64; do
+          for t in linux/amd64 windows/amd64; do
             echo "building for $t"
             GOOS=${t%/*} GOARCH=${t#*/} GOARM=7 CGO_ENABLED=0 go build -o /dev/null ./cmd/blacktip
           done

.gitea/workflows/release.yml

@@ -59,4 +59,4 @@ jobs:
             *-*) ;;                       # pre-releases (v1.0.0-rc1) don't move :latest
             *) tags="$tags -t $image:latest" ;;
           esac
-          docker buildx build --platform linux/amd64,linux/arm64 $tags --push .
+          docker buildx build --platform linux/amd64 $tags --push .

README.md

@@ -38,7 +38,7 @@ Settings and secrets can live in a `.env` file instead of the command line; see
 
 ## Deploying
 
-Releases are built by Gitea Actions. Pushing a tag such as `v0.2.0` runs `.gitea/workflows/release.yml`, which tests the code, builds Blacktip for Linux (amd64, arm64, armv7), Windows (amd64) and macOS (amd64, arm64), publishes the archives with a `SHA256SUMS` file on the Releases page, and pushes a multi-architecture container image to this server's registry (`git.veillax.com/veillax/blacktip`). Every push and pull request runs `.gitea/workflows/ci.yml`: formatting, vet, tests, and a check that every release platform still compiles.
+Releases are built by Gitea Actions. Pushing a tag such as `v0.2.0` runs `.gitea/workflows/release.yml`, which tests the code, builds Blacktip for Linux (amd64) and Windows (amd64), publishes the archives with a `SHA256SUMS` file on the Releases page, and pushes a container image (linux/amd64) to this server's registry (`git.veillax.com/veillax/blacktip`). Every push and pull request runs `.gitea/workflows/ci.yml`: formatting, vet, tests, and a check that every release platform still compiles.
 
 **On a Linux server (systemd)**
 
@@ -52,7 +52,7 @@ sudoedit /etc/blacktip/blacktip.env   # your settings and secrets
 sudo systemctl start blacktip
 ```
 
-Use `linux_arm64` or `linux_arm` for Raspberry Pi and other ARM machines. To upgrade, repeat the same steps: your settings and data are kept. `sudo ./install.sh --uninstall` removes the service and binary but leaves `/etc/blacktip` and `/var/lib/blacktip` alone. `blacktip -version` tells you which build is running.
+Releases are published for x86-64 only; for ARM machines build it yourself (`TARGETS=linux/arm64 sh deploy/build.sh`). To upgrade, repeat the same steps: your settings and data are kept. `sudo ./install.sh --uninstall` removes the service and binary but leaves `/etc/blacktip` and `/var/lib/blacktip` alone. `blacktip -version` tells you which build is running.
 
 **Updates**
 

blacktip.go

@@ -38,6 +38,13 @@ type Config struct {
 	DataDir    string
 	Docker     bool // report local Docker containers for discovery
 
+	// Version is the release this binary was built from, reported to Hammerhead. AutoUpdate says
+	// whether the program may update itself; only then does it act on an update request from
+	// Hammerhead, by calling UpdateNow (which normally restarts the process into the new binary).
+	Version    string
+	AutoUpdate bool
+	UpdateNow  func(ctx context.Context, progress func(stage string, pct int, line string)) (newVersion string, err error)
+
 	// Traefik, when its Path is set, serves http/https resources instead of the built-in proxy.
 	Traefik TraefikConfig
 
@@ -106,6 +113,13 @@ type Blacktip struct {
 
 	logq       chan model.LogEntry
 	reportMu   sync.Mutex
+	updating   atomic.Bool
+	updateMsg  string // outcome of the update Hammerhead asked for, reported back
+	updateID   string // the request that updateMsg answers
+	updStage   string // progress of the running update, reported to Hammerhead
+	updPct     int
+	updLog     []string
+	updSent    time.Time
 	dockerMu   sync.Mutex
 	dockerAt   time.Time
 	dockerList []model.Container
@@ -252,6 +266,9 @@ func (b *Blacktip) apply(cfg *model.EdgeConfig) {
 	if b.dashboard == "" {
 		b.dashboard = b.cfg.Hammerhead
 	}
+	if cfg.UpdateID != "" {
+		b.requestUpdate(cfg.UpdateID)
+	}
 	b.local = map[string]bool{}
 	for _, id := range cfg.Local {
 		b.local[id] = true
@@ -517,8 +534,11 @@ func (b *Blacktip) report() {
 		IPs: localIPs(), PublicKey: b.wgPub,
 		Status: map[string]model.MappingStatus{}, Stats: map[string]model.Stat{},
 		Containers: b.containers(), MailChecks: map[string]model.MailCheck{},
+		Version: b.cfg.Version, AutoUpdate: b.cfg.AutoUpdate,
 	}
 	b.mu.Lock()
+	rep.UpdateMsg, rep.UpdateID = b.updateMsg, b.updateID
+	rep.UpdateStage, rep.UpdatePct, rep.UpdateLog = b.updStage, b.updPct, append([]string(nil), b.updLog...)
 	for id, chk := range b.mailChecks {
 		rep.MailChecks[id] = chk
 	}
@@ -736,3 +756,73 @@ func (ms *mapState) dialOff(proto string, prefer, off int) (net.Conn, bool, erro
 	}
 	return nil, false, last
 }
+
+// requestUpdate acts on "update now" from Hammerhead (id identifies the request). It only ever installs
+// what this machine's own auto-update would (the newest release from its configured server), and only
+// when auto-update is on. Caller holds b.mu.
+func (b *Blacktip) requestUpdate(id string) {
+	if id == b.updateID {
+		return // already handled, or in progress
+	}
+	b.updateID, b.updateMsg = id, ""
+	b.updStage, b.updPct, b.updLog = "starting", 0, nil
+	b.updNote("Update requested from the dashboard")
+	if !b.cfg.AutoUpdate || b.cfg.UpdateNow == nil {
+		b.updateMsg = "auto-update is off on this machine (set BLACKTIP_AUTO_UPDATE=1 and restart it)"
+		b.updStage = "failed"
+		b.updNote("Refused: " + b.updateMsg)
+		go b.report()
+		return
+	}
+	if !b.updating.CompareAndSwap(false, true) {
+		b.updateMsg = "another update is already running"
+		b.updStage = "failed"
+		b.updNote("Refused: " + b.updateMsg)
+		go b.report()
+		return
+	}
+	go func() {
+		defer b.updating.Store(false)
+		tag, err := b.cfg.UpdateNow(b.ctx, b.updProgress) // on success this restarts the process and does not return
+		b.mu.Lock()
+		switch {
+		case err != nil:
+			b.updateMsg = "update failed: " + err.Error()
+		case tag == "":
+			b.updateMsg = "already on the newest release"
+		default:
+			b.updateMsg = "updated to " + tag + ", restarting"
+		}
+		b.mu.Unlock()
+		b.report()
+	}()
+}
+
+// updNote adds a timestamped line to the update log. Caller holds b.mu.
+func (b *Blacktip) updNote(line string) {
+	b.updLog = append(b.updLog, time.Now().Format("15:04:05")+" "+line)
+	if len(b.updLog) > 200 {
+		b.updLog = b.updLog[len(b.updLog)-200:]
+	}
+}
+
+// updProgress is called by the updater as it works; Hammerhead gets a fresh report at most once a
+// second (and straight away for every new log line), so its progress bar follows along.
+func (b *Blacktip) updProgress(stage string, pct int, line string) {
+	b.mu.Lock()
+	b.updStage = stage
+	if pct >= 0 {
+		b.updPct = pct
+	}
+	if line != "" {
+		b.updNote(line)
+	}
+	send := line != "" || time.Since(b.updSent) > time.Second
+	if send {
+		b.updSent = time.Now()
+	}
+	b.mu.Unlock()
+	if send {
+		go b.report()
+	}
+}

cmd/blacktip/main.go

@@ -50,9 +50,16 @@ func main() {
 		flag.Usage()
 		os.Exit(2)
 	}
+	upd := selfupdate.Config{Server: "https://git.veillax.com", Repo: "Veillax/blacktip", App: "blacktip", Current: version, Logf: log.Printf}
 	b, err := blacktip.New(blacktip.Config{
 		Hammerhead: strings.TrimRight(*hh, "/"), Token: *token, TunnelAddr: *tun, DataDir: *data, Docker: *dockerOn,
 		Traefik: blacktip.TraefikConfig{Path: *traefikPath, ACMEDNS: *acmeDNS, NoACME: *noACME},
+		Version: version, AutoUpdate: *autoUpdate,
+		UpdateNow: func(ctx context.Context, progress func(string, int, string)) (string, error) {
+			c := upd
+			c.Progress = func(e selfupdate.Event) { progress(e.Stage, e.Pct, e.Line) }
+			return selfupdate.CheckOnce(ctx, c)
+		},
 	})
 	if err != nil {
 		log.Fatal(err)
@@ -61,7 +68,7 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 	if *autoUpdate {
-		go selfupdate.Run(ctx, selfupdate.Config{Server: "https://git.veillax.com", Repo: "Veillax/blacktip", App: "blacktip", Current: version, Logf: log.Printf})
+		go selfupdate.Run(ctx, upd)
 	}
 	if err := b.Run(ctx); err != nil {
 		log.Fatal(err)

deploy/build.sh

@@ -8,12 +8,13 @@
 #   dist/blacktip_<version>_<os>_<arch>.tar.gz or .zip
 #   dist/SHA256SUMS
 #
-# Set TARGETS="linux/amd64 linux/arm64" to build fewer platforms.
+# Releases are published for linux/amd64 and windows/amd64 only. Set TARGETS="linux/arm64 darwin/arm64"
+# (any GOOS/GOARCH pair Go supports) to build something else by hand.
 set -eu
 NAME=blacktip
 cd "$(dirname "$0")/.."
 VERSION=${1:-$(git describe --tags --always 2>/dev/null || echo dev)}
-TARGETS=${TARGETS:-"linux/amd64 linux/arm64 linux/arm windows/amd64 darwin/amd64 darwin/arm64"}
+TARGETS=${TARGETS:-"linux/amd64 windows/amd64"}
 
 rm -rf dist
 mkdir -p dist/bin dist/stage

deploy/install.sh

@@ -51,7 +51,9 @@ mkdir -p "$LIB"
 install -m 0755 "$HERE/$NAME" "$LIB/$NAME.new"
 mv -f "$LIB/$NAME.new" "$LIB/$NAME" # atomic, so upgrading a running service is safe
 rm -f "$BIN"
-ln -s "/usr/local/lib/$NAME/$NAME" "$BIN"
+LINK=/usr/local/lib/$NAME/$NAME
+[ -z "${DESTDIR:-}" ] || LINK=$LIB/$NAME # staging: point inside the staging folder
+ln -s "$LINK" "$BIN"
 if [ -f "$HERE/traefik" ]; then
 	install -m 0755 "$HERE/traefik" "$ROOT/usr/local/lib/$NAME/traefik.new"
 	mv -f "$ROOT/usr/local/lib/$NAME/traefik.new" "$ROOT/usr/local/lib/$NAME/traefik"

deploy/publish-release.sh

@@ -42,19 +42,30 @@ else
 	echo "release $TAG already exists (id $id): replacing its files"
 fi
 
-existing=$(curl -fsS -H "$AUTH" "$API/releases/$id/assets" | json 'import sys, json
+# the ids of the release's assets with a given name (a failed upload can leave a partial one behind)
+asset_ids() {
+	curl -fsS -H "$AUTH" "$API/releases/$id/assets" | "$PY" -c 'import sys, json
 for a in json.load(sys.stdin):
-    print(a["id"], a["name"])')
+    if a["name"] == sys.argv[1]:
+        print(a["id"])' "$1"
+}
 
 for f in dist/*.tar.gz dist/*.zip dist/SHA256SUMS; do
 	[ -f "$f" ] || continue
 	name=$(basename "$f")
-	echo "$existing" | while read -r aid aname; do
-		if [ "$aname" = "$name" ]; then
+	n=1
+	while :; do
+		for aid in $(asset_ids "$name"); do
 			curl -fsS -X DELETE -H "$AUTH" "$API/releases/$id/assets/$aid" > /dev/null
-		fi
 		done
-	echo "uploading $name"
-	curl -fsS -X POST -H "$AUTH" -F "attachment=@$f" "$API/releases/$id/assets?name=$name" > /dev/null
+		echo "uploading $name (attempt $n)"
+		if curl -fsS --max-time 1200 -X POST -H "$AUTH" -F "attachment=@$f" "$API/releases/$id/assets?name=$name" > /dev/null; then
+			break
+		fi
+		# large uploads can hit a proxy timeout (HTTP 502); try again
+		[ "$n" -lt 5 ] || { echo "giving up on $name after $n attempts" >&2; exit 1; }
+		n=$((n + 1))
+		sleep $((n * 10))
+	done
 done
 echo "published $SERVER/$REPO/releases/tag/$TAG"

go.mod

@@ -3,7 +3,7 @@ module git.veillax.com/Veillax/blacktip
 go 1.27.3
 
 require (
-	git.veillax.com/Veillax/reef v0.1.3
+	git.veillax.com/Veillax/reef v0.1.7
 	github.com/emersion/go-msgauth v0.7.0
 	golang.org/x/crypto v0.58.0
 )

go.sum

@@ -2,6 +2,12 @@ git.veillax.com/Veillax/reef v0.1.0 h1:O0OhEKFx2CsvACJ20sfzJNjGsngomtfDdKdxu4ROp
 git.veillax.com/Veillax/reef v0.1.0/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
 git.veillax.com/Veillax/reef v0.1.3 h1:RxSq2xwaup1+MieAkhlvOTZnYwPf19thFHEAe/7VxsI=
 git.veillax.com/Veillax/reef v0.1.3/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
+git.veillax.com/Veillax/reef v0.1.5 h1:BydjLJsfGgU4zEIqUsePJRjPi9NgbFdmvq0M8dvJCgM=
+git.veillax.com/Veillax/reef v0.1.5/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
+git.veillax.com/Veillax/reef v0.1.6 h1:NgZ4u2Z3hJ+BYD/zX9lov8fox4s2i8txtYNOpIsikVo=
+git.veillax.com/Veillax/reef v0.1.6/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
+git.veillax.com/Veillax/reef v0.1.7 h1:5C2udonlJac2xvDOE5e+zDxej+AbZ9ttrNSE4owmwuU=
+git.veillax.com/Veillax/reef v0.1.7/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
 github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g=
 github.com/Microsoft/go-winio v0.6.4/go.mod h1:1xl5JXG4ze/m74GH6Dc4Mwxfc3VL5siRVQjIAlHH07U=
 github.com/emersion/go-msgauth v0.7.0 h1:vj2hMn6KhFtW41kshIBTXvp6KgYSqpA/ZN9Pv4g1INc=