Show a progress bar and live log while Hammerhead, edges and sites update

The updater reports each step (checking, downloading with percent, verifying, installing, restarting).
Edges and sites send that progress and their log with their reports; Hammerhead serves it to a dialog
that opens when an update starts, and keeps its own last update log on disk across the restart.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

README.md

@@ -73,7 +73,7 @@ Releases are published for x86-64 only; for ARM machines build it yourself (`TAR
 
 Set `HAMMERHEAD_AUTO_UPDATE=1` (or pass `-auto-update`) and Hammerhead checks this server for a newer release every few hours, verifies it against `SHA256SUMS`, checks that the new binary runs, swaps it in and restarts itself. It only follows full releases (never pre-releases), never downgrades, and keeps the old binary if anything looks wrong. `setup.sh` turns it on. It is not available on Windows, and it is for the systemd and plain-binary installs: with Docker, pull a new image instead.
 
-Edges and sites report their version, so Server → Edges and Sites show which release each one runs and flag a newer one. An **Update** button appears for any machine that runs with auto-update on (`BLACKTIP_AUTO_UPDATE=1` / `BAMBOO_AUTO_UPDATE=1`): it asks that machine to install the newest release now, and the result is shown in the table. Under Server → Settings, **Check for updates** does the same lookup for Hammerhead itself, with an **Update now** button when auto-update is on. A machine without auto-update never installs anything on request.
+Edges and sites report their version, so Server → Edges and Sites show which release each one runs and flag a newer one. An **Update** button appears for any machine that runs with auto-update on (`BLACKTIP_AUTO_UPDATE=1` / `BAMBOO_AUTO_UPDATE=1`): it asks that machine to install the newest release now, and the result is shown in the table. Under Server → Settings, **Check for updates** does the same lookup for Hammerhead itself, with an **Update now** button when auto-update is on. A machine without auto-update never installs anything on request. Starting an update opens a progress bar with the live log of what the machine is doing (checking, downloading, verifying, installing, restarting). Hammerhead keeps the log of its own last update in `update.log` in its data folder, so it is still there after the restart.
 
 **With Docker**
 

agent.go

@@ -111,7 +111,9 @@ func (s *Server) edgeReport(w http.ResponseWriter, r *http.Request) {
 	}
 	e.LastSeen = time.Now()
 	e.IPs = in.IPs
+	oldVersion := e.Version
 	answered := noteUpdateReport(&e.Version, &e.AutoUpdate, &e.UpdateID, &e.UpdateMsg, in.Version, in.AutoUpdate, in.UpdateID, in.UpdateMsg)
+	s.recordUpdateProgress(e.ID, in.UpdateID, in.UpdateStage, in.UpdatePct, in.UpdateLog, oldVersion, e.Version)
 	for _, x := range s.st.MailEgress {
 		if x.EdgeID == e.ID {
 			if chk, found := in.MailChecks[x.ID]; found {
@@ -253,7 +255,9 @@ func (s *Server) siteReport(w http.ResponseWriter, r *http.Request) {
 		in.Connected = []string{}
 	}
 	st.Connected = in.Connected
+	oldVersion := st.Version
 	answered := noteUpdateReport(&st.Version, &st.AutoUpdate, &st.UpdateID, &st.UpdateMsg, in.Version, in.AutoUpdate, in.UpdateID, in.UpdateMsg)
+	s.recordUpdateProgress(st.ID, in.UpdateID, in.UpdateStage, in.UpdatePct, in.UpdateLog, oldVersion, st.Version)
 	s.containers[st.ID] = in.Containers
 	for _, x := range s.st.MailEgress {
 		if x.SiteID != st.ID {

e2e/updates_test.go

@@ -58,12 +58,23 @@ func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) {
 	var edgeCalls, siteCalls atomic.Int32
 	siteCfgHook = func(c *bamboo.Config) {
 		c.Version, c.AutoUpdate = "v0.1.0", true
-		c.UpdateNow = func(ctx context.Context) (string, error) { siteCalls.Add(1); return "", nil }
+		c.UpdateNow = func(ctx context.Context, progress func(string, int, string)) (string, error) {
+			siteCalls.Add(1)
+			progress("checking", 10, "Checking for the newest release")
+			progress("done", 100, "Already on the newest release")
+			return "", nil
+		}
 	}
 	defer func() { siteCfgHook = nil }()
 	w := setup(t, blacktip.TraefikConfig{}, func(c *blacktip.Config) {
 		c.Version, c.AutoUpdate = "v0.1.0", true
-		c.UpdateNow = func(ctx context.Context) (string, error) { edgeCalls.Add(1); return "v0.2.0", nil }
+		c.UpdateNow = func(ctx context.Context, progress func(string, int, string)) (string, error) {
+			edgeCalls.Add(1)
+			progress("downloading", 40, "Downloading blacktip_v0.2.0_linux_amd64.tar.gz")
+			progress("downloading", 70, "Downloaded 28.0 of 40.0 MB (70%)")
+			progress("restarting", 100, "Installed v0.2.0; restarting into it")
+			return "v0.2.0", nil
+		}
 	})
 
 	waitFor(t, "the edge to report its version", func() bool { e := w.edgeRow(); return e.Version == "v0.1.0" && e.AutoUpdate })
@@ -77,6 +88,17 @@ func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) {
 	if edgeCalls.Load() != 1 {
 		t.Fatalf("the edge ran its update %d times, want 1", edgeCalls.Load())
 	}
+	var log struct {
+		Running bool
+		Stage   string
+		Pct     int
+		Lines   []string
+	}
+	w.admin.must("GET", "/api/edges/"+w.edge+"/update/log", nil, &log)
+	text := strings.Join(log.Lines, "; ")
+	if log.Running || log.Stage != "restarting" || log.Pct != 100 || !strings.Contains(text, "Update requested") || !strings.Contains(text, "Downloaded 28.0 of 40.0 MB") || !strings.Contains(text, "restarting into it") {
+		t.Fatalf("edge update log: %+v (%s)", log, text)
+	}
 
 	w.admin.must("POST", "/api/orgs/"+w.org+"/sites/"+w.site+"/update", nil, nil)
 	waitFor(t, "the site to answer the update request", func() bool {
@@ -86,6 +108,10 @@ func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) {
 	if siteCalls.Load() != 1 {
 		t.Fatalf("the site ran its update %d times, want 1", siteCalls.Load())
 	}
+	w.admin.must("GET", "/api/orgs/"+w.org+"/sites/"+w.site+"/update/log", nil, &log)
+	if log.Running || log.Stage != "done" || !strings.Contains(strings.Join(log.Lines, "; "), "Already on the newest release") {
+		t.Fatalf("site update log: %+v", log)
+	}
 
 	// asking again works and is a fresh request, not a leftover answer
 	w.admin.must("POST", "/api/edges/"+w.edge+"/update", nil, nil)

go.mod

@@ -3,9 +3,9 @@ module git.veillax.com/Veillax/hammerhead
 go 1.27.3
 
 require (
-	git.veillax.com/Veillax/bamboo v0.1.4
+	git.veillax.com/Veillax/bamboo v0.1.5
-	git.veillax.com/Veillax/blacktip v0.1.6
+	git.veillax.com/Veillax/blacktip v0.1.7
-	git.veillax.com/Veillax/reef v0.1.6
+	git.veillax.com/Veillax/reef v0.1.7
 	github.com/emersion/go-msgauth v0.7.0
 	golang.org/x/crypto v0.58.0
 	modernc.org/sqlite v1.61.0

go.sum

@@ -4,12 +4,16 @@ git.veillax.com/Veillax/bamboo v0.1.3 h1:A66gz9GmmtUq1iIGUAL2ItIeC6iTy5pvqJOoIjR
 git.veillax.com/Veillax/bamboo v0.1.3/go.mod h1:UuHV6aNHmV9jpow0xeGlXX0QhMB9glPJUGzgIM7S1MM=
 git.veillax.com/Veillax/bamboo v0.1.4 h1:pgiBJA46CBJx7T3bloCjPekb6wl6W5H8teokFAH5gj0=
 git.veillax.com/Veillax/bamboo v0.1.4/go.mod h1:wzSpfql7U4a3b9mCVH01OPG9H9szrzwSFDufWw02e0g=
+git.veillax.com/Veillax/bamboo v0.1.5 h1:i81c75/7wjayP9tOQ3tSnJWTBhvbjAwNBEPhT+3P2Xk=
+git.veillax.com/Veillax/bamboo v0.1.5/go.mod h1:se4kkcP6c3uCGD4HQd+nDLoblS0lN8qa+eFWLo5Horw=
 git.veillax.com/Veillax/blacktip v0.1.0 h1:L8WpW56gqtvZ1IAapE95Ynl6NKjqTA98WwmEH01kFFk=
 git.veillax.com/Veillax/blacktip v0.1.0/go.mod h1:sj+mr8393rajzWbl0+RRZrstGyDtZ/GHsut6ebScVxg=
 git.veillax.com/Veillax/blacktip v0.1.4 h1:o3uNoA0ucB/qcN9ccofysdFA6RTEU4jKdgeTWuE2Uw0=
 git.veillax.com/Veillax/blacktip v0.1.4/go.mod h1:f7zXCYlQf/9ffDLjPrMHylKAyXX4Nb+aQ7kErmOOhqI=
 git.veillax.com/Veillax/blacktip v0.1.6 h1:dUJzNGXgieWX05ftWjnMfh6norhaeKTqo1P13lJvjtk=
 git.veillax.com/Veillax/blacktip v0.1.6/go.mod h1:ZnPRc2gPh9ElILfPH39G/x3B6KAiHQyeoDLycSDgVPM=
+git.veillax.com/Veillax/blacktip v0.1.7 h1:PjxEkhE4fnGWht9Pj2/l6S8fOX2Iqjir3rc9vfpMFiQ=
+git.veillax.com/Veillax/blacktip v0.1.7/go.mod h1:dVm4wC6wufPjFzK8pvHnqNJJlxroRYmnq8mdkCqOKGc=
 git.veillax.com/Veillax/reef v0.1.0 h1:O0OhEKFx2CsvACJ20sfzJNjGsngomtfDdKdxu4ROpCs=
 git.veillax.com/Veillax/reef v0.1.0/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
 git.veillax.com/Veillax/reef v0.1.3 h1:RxSq2xwaup1+MieAkhlvOTZnYwPf19thFHEAe/7VxsI=
@@ -18,6 +22,8 @@ git.veillax.com/Veillax/reef v0.1.4 h1:T2+B8P1GAitEyTYRkwMT3UHbG9mN5zXb/inE2muXd
 git.veillax.com/Veillax/reef v0.1.4/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
 git.veillax.com/Veillax/reef v0.1.6 h1:NgZ4u2Z3hJ+BYD/zX9lov8fox4s2i8txtYNOpIsikVo=
 git.veillax.com/Veillax/reef v0.1.6/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
+git.veillax.com/Veillax/reef v0.1.7 h1:5C2udonlJac2xvDOE5e+zDxej+AbZ9ttrNSE4owmwuU=
+git.veillax.com/Veillax/reef v0.1.7/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
 github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g=
 github.com/Microsoft/go-winio v0.6.4/go.mod h1:1xl5JXG4ze/m74GH6Dc4Mwxfc3VL5siRVQjIAlHH07U=
 github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=

server.go

@@ -99,7 +99,10 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("POST /api/admin/settings/test-mail", s.withServerAdmin(s.testMail))
 	mux.HandleFunc("GET /api/admin/update", s.withServerAdmin(s.checkUpdate))
 	mux.HandleFunc("POST /api/admin/update/apply", s.withServerAdmin(s.applyUpdate))
+	mux.HandleFunc("GET /api/admin/update/status", s.withServerAdmin(s.updateStatus))
 	mux.HandleFunc("GET /api/versions", s.withUser(s.latestVersions))
+	mux.HandleFunc("GET /api/edges/{id}/update/log", s.withServerAdmin(s.edgeUpdateLog))
+	mux.HandleFunc("GET /api/orgs/{org}/sites/{id}/update/log", s.withOrg(true, s.siteUpdateLog))
 	mux.HandleFunc("POST /api/edges/{id}/update", s.withServerAdmin(s.updateEdge))
 	mux.HandleFunc("POST /api/orgs/{org}/sites/{id}/update", s.withOrg(true, s.updateSite))
 	mux.HandleFunc("GET /api/admin/audit", s.withServerAdmin(func(w http.ResponseWriter, r *http.Request, p *principal) {

store.go

@@ -209,6 +209,8 @@ type Server struct {
 	oidc    map[string]*oidcDoc
 
 	alerts   map[string]*alertState // active problems by key (see alerts.go)
+	updRuns  map[string]*updRun     // progress of updates running on edges and sites, by ID
+	selfUpd  selfUpdate             // progress of this server's own update
 	verMu    sync.Mutex             // guards verCache and verAt, the latest releases seen by /api/versions
 	verCache map[string]string
 	verAt    time.Time
@@ -253,6 +255,7 @@ func New(dir string, opts Options) (*Server, error) {
 		oidc:       map[string]*oidcDoc{},
 		alerts:     map[string]*alertState{},
 		mtaSent:    map[string]time.Time{},
+		updRuns:    map[string]*updRun{},
 	}
 	s.st.Sessions = map[string]*Session{}
 	s.st.Secrets = map[string]*Secret{}
@@ -282,6 +285,7 @@ func New(dir string, opts Options) (*Server, error) {
 	if opts.SelfDomain != "" && opts.SelfTunnel == "" {
 		s.opts.SelfTunnel = opts.SelfDomain + ":51820"
 	}
+	s.finishSelfUpdateLog()
 	if err := s.ensureSelfEdge(); err != nil {
 		db.Close()
 		return nil, err

update.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"net/http"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -45,13 +46,32 @@ func (s *Server) applyUpdate(w http.ResponseWriter, r *http.Request, p *principa
 		fail(w, 400, errors.New("auto-update is off: set HAMMERHEAD_AUTO_UPDATE=1 and restart Hammerhead, or update it by hand"))
 		return
 	}
+	u := &s.selfUpd
+	u.mu.Lock()
+	if u.running {
+		u.mu.Unlock()
+		fail(w, 409, errors.New("an update is already running"))
+		return
+	}
+	u.running, u.stage, u.pct, u.lines = true, "starting", 0, nil
+	u.mu.Unlock()
+	os.WriteFile(s.updateLogPath(), nil, 0o600) // a new run starts a new log
 	cfg := s.updateConfig("hammerhead", s.opts.Version)
-	cfg.Restart = func(exe string) error { s.Close(); return selfupdate.Restart(exe) }
+	cfg.Progress = func(e selfupdate.Event) { s.selfNote(e.Stage, e.Pct, e.Line) }
+	cfg.Restart = func(exe string) error {
+		s.selfNote("restarting", 100, "Closing the database and restarting")
+		s.Close()
+		return selfupdate.Restart(exe)
+	}
+	s.selfNote("starting", 0, "Update requested by "+p.User.Email)
 	s.audit("", p.User.Email, "server.update", "")
 	go func() {
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 		defer cancel()
 		selfupdate.CheckOnce(ctx, cfg) // on success the process is replaced and this never returns
+		u.mu.Lock()
+		u.running = false
+		u.mu.Unlock()
 	}()
 	writeJSON(w, 200, map[string]any{"started": true})
 }
@@ -121,6 +141,7 @@ func (s *Server) updateEdge(w http.ResponseWriter, r *http.Request, p *principal
 		fail(w, 400, err)
 		return
 	}
+	delete(s.updRuns, e.ID) // a new request starts a new log
 	s.bump()
 	s.audit("", p.User.Email, "edge.update", e.Name)
 	ok(w)
@@ -142,6 +163,7 @@ func (s *Server) updateSite(w http.ResponseWriter, r *http.Request, c *orgCtx) {
 		fail(w, 400, err)
 		return
 	}
+	delete(s.updRuns, st.ID)
 	s.bump()
 	s.audit(c.Org.ID, c.Actor, "site.update", st.Name)
 	ok(w)

update_test.go

@@ -5,6 +5,9 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -52,3 +55,36 @@ func TestCheckForUpdates(t *testing.T) {
 		t.Errorf("unreachable server: %d %s", code, b)
 	}
 }
+
+func TestSelfUpdateLogSurvivesTheRestartItEndsWith(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "update.log"), []byte("12:00:00 Checking\n12:00:09 Installed v0.3.0; restarting into it\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	s, err := New(dir, Options{AdminEmail: "admin@test.local", AdminPassword: "password123", MasterKey: key, Version: "v0.3.0"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	do := apiClient(t, s)
+	code, b := do("GET", "/api/admin/update/status", nil)
+	var out struct {
+		Running bool
+		Stage   string
+		Pct     int
+		Lines   []string
+		Version string
+	}
+	json.Unmarshal(b, &out)
+	last := ""
+	if len(out.Lines) > 0 {
+		last = out.Lines[len(out.Lines)-1]
+	}
+	if code != 200 || out.Running || out.Stage != "done" || out.Pct != 100 || len(out.Lines) != 3 || !strings.Contains(last, "now running v0.3.0") {
+		t.Fatalf("status: %d %s", code, b)
+	}
+	// starting an update needs auto-update on
+	if code, b := do("POST", "/api/admin/update/apply", nil); code != 400 || !strings.Contains(string(b), "HAMMERHEAD_AUTO_UPDATE") {
+		t.Fatalf("apply without auto-update: %d %s", code, b)
+	}
+}

updatelog.go

@@ -0,0 +1,152 @@
+package hammerhead
+
+import (
+	"errors"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+const maxUpdateLines = 200
+
+// updRun is the progress of one update an edge or site is carrying out, as that agent reported it.
+type updRun struct {
+	ID    string // the update request it belongs to
+	Stage string
+	Pct   int
+	Lines []string
+}
+
+// recordUpdateProgress keeps what an agent says about the update it is running. When the agent
+// reports a new version, the update is finished: its log gets a closing line and is then left alone,
+// so the freshly restarted program (which starts a new, empty log) can't overwrite it.
+// Caller holds s.mu.
+func (s *Server) recordUpdateProgress(agentID, reqID, stage string, pct int, lines []string, oldVersion, newVersion string) {
+	run := s.updRuns[agentID]
+	if run != nil && run.Stage != "done" && oldVersion != "" && newVersion != "" && newVersion != oldVersion {
+		run.Lines = append(run.Lines, time.Now().Format("15:04:05")+" Now running "+newVersion)
+		run.Stage, run.Pct = "done", 100
+		return
+	}
+	if reqID == "" || (stage == "" && len(lines) == 0) {
+		return
+	}
+	if run != nil && run.ID == reqID && run.Stage == "done" {
+		return
+	}
+	s.updRuns[agentID] = &updRun{ID: reqID, Stage: stage, Pct: pct, Lines: lines}
+}
+
+func (s *Server) updateLogView(agentID string, pending bool, msg, version string) map[string]any {
+	out := map[string]any{"running": pending, "msg": msg, "version": version, "stage": "", "pct": 0, "lines": []string{}}
+	if run := s.updRuns[agentID]; run != nil {
+		out["stage"], out["pct"], out["lines"] = run.Stage, run.Pct, run.Lines
+	}
+	return out
+}
+
+func (s *Server) edgeUpdateLog(w http.ResponseWriter, r *http.Request, p *principal) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	e := s.findEdge(r.PathValue("id"))
+	if e == nil {
+		fail(w, 404, errors.New("no such edge"))
+		return
+	}
+	writeJSON(w, 200, s.updateLogView(e.ID, e.UpdateID != "", e.UpdateMsg, e.Version))
+}
+
+func (s *Server) siteUpdateLog(w http.ResponseWriter, r *http.Request, c *orgCtx) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	st := s.findSite(r.PathValue("id"))
+	if st == nil || !s.siteVisible(c, st) {
+		fail(w, 404, errors.New("no such site"))
+		return
+	}
+	writeJSON(w, 200, s.updateLogView(st.ID, st.UpdateID != "", st.UpdateMsg, st.Version))
+}
+
+// selfUpdate is the progress of this server's own update. It is kept in memory and in <data>/update.log,
+// so the page can still show it after the restart that ends a successful update.
+type selfUpdate struct {
+	mu      sync.Mutex
+	running bool
+	stage   string
+	pct     int
+	lines   []string
+}
+
+func (s *Server) updateLogPath() string { return filepath.Join(s.dir, "update.log") }
+
+func (s *Server) selfNote(stage string, pct int, line string) {
+	u := &s.selfUpd
+	u.mu.Lock()
+	defer u.mu.Unlock()
+	if stage != "" {
+		u.stage = stage
+	}
+	if pct >= 0 {
+		u.pct = pct
+	}
+	if line == "" {
+		return
+	}
+	line = time.Now().Format("15:04:05") + " " + line
+	u.lines = append(u.lines, line)
+	if len(u.lines) > maxUpdateLines {
+		u.lines = u.lines[len(u.lines)-maxUpdateLines:]
+	}
+	if f, err := os.OpenFile(s.updateLogPath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600); err == nil {
+		f.WriteString(line + "\n")
+		f.Close()
+	}
+}
+
+func readLines(path string) []string {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	lines := strings.Split(strings.TrimRight(string(b), "\n"), "\n")
+	if len(lines) == 1 && lines[0] == "" {
+		return nil
+	}
+	if len(lines) > maxUpdateLines {
+		lines = lines[len(lines)-maxUpdateLines:]
+	}
+	return lines
+}
+
+// finishSelfUpdateLog runs at startup: if the last run ended by restarting into this process, say so.
+func (s *Server) finishSelfUpdateLog() {
+	lines := readLines(s.updateLogPath())
+	if n := len(lines); n > 0 && strings.Contains(lines[n-1], "restarting") && s.opts.Version != "" {
+		if f, err := os.OpenFile(s.updateLogPath(), os.O_APPEND|os.O_WRONLY, 0o600); err == nil {
+			f.WriteString(time.Now().Format("15:04:05") + " Hammerhead restarted: now running " + s.opts.Version + "\n")
+			f.Close()
+		}
+	}
+}
+
+// updateStatus returns the progress and log of this server's own update, for the Settings page.
+func (s *Server) updateStatus(w http.ResponseWriter, r *http.Request, p *principal) {
+	u := &s.selfUpd
+	u.mu.Lock()
+	out := map[string]any{"running": u.running, "stage": u.stage, "pct": u.pct, "lines": append([]string(nil), u.lines...), "version": s.opts.Version}
+	u.mu.Unlock()
+	if lines, _ := out["lines"].([]string); len(lines) == 0 {
+		// nothing in memory: this is a fresh process, so show the last update's log from disk
+		if saved := readLines(s.updateLogPath()); len(saved) > 0 {
+			stage := "done"
+			if strings.Contains(saved[len(saved)-1], "Failed:") {
+				stage = "failed"
+			}
+			out["lines"], out["stage"], out["pct"] = saved, stage, 100
+		}
+	}
+	writeJSON(w, 200, out)
+}

web/index.html

@@ -76,6 +76,10 @@
   .choice.on { border-color:var(--accent); background:var(--accent-soft); }
   .choice-t { font-weight:600; }
   .sect-h { padding:var(--s3) var(--s4); font-weight:600; border-bottom:1px solid var(--border); }
+  .progress { height:8px; background:var(--surface-2); border-radius:999px; overflow:hidden; }
+  .progress > div { height:100%; background:var(--accent); transition:width .4s ease; }
+  .progress > div.bad { background:var(--danger); }
+  .logbox { margin:0; background:var(--bg); border:1px solid var(--border); border-radius:var(--radius); padding:var(--s3); font:var(--text-xs)/1.6 var(--font-mono); max-height:260px; overflow:auto; white-space:pre-wrap; word-break:break-word; }
 
   /* forms */
   label { display:flex; flex-direction:column; gap:var(--s1); font-size:var(--text-sm); font-weight:600; color:var(--text); }
@@ -191,6 +195,34 @@ const byId = (list, id) => (list || []).find(x => x.id === id);
 const field = (label, el, hint) => h('label', {}, label, el, hint ? h('span', {class:'sub'}, hint) : null);
 const chk = (label, checked, onchange) => { const i = h('input', {type:'checkbox', checked: !!checked}); if (onchange) i.onchange = () => onchange(i.checked); return Object.assign(h('label', {class:'chk'}, i, label), {box:i}); };
 // ----- versions and updates -----
+// A progress bar and live log for an update. load() returns {running, stage, pct, lines}; the panel polls it
+// until the update is over, and keeps going through the gap while a program restarts into its new version.
+function updatePanel(load, onDone) {
+  const bar = h('div', {style:'width:0%'}), stageEl = h('div', {class:'small'}, 'Starting…'), logEl = h('pre', {class:'logbox'}, 'Waiting for the first message…');
+  const el = h('div', {class:'stack'}, stageEl, h('div', {class:'progress'}, bar), logEl);
+  const label = {starting:'Starting', checking:'Checking for the release', downloading:'Downloading', verifying:'Verifying', installing:'Installing', restarting:'Restarting', done:'Done', failed:'Failed'};
+  let misses = 0;
+  const tick = async () => {
+    if (!el.isConnected) return; // the dialog or card was closed
+    let again = true;
+    try {
+      const s = await load(); misses = 0;
+      const pct = Math.max(0, Math.min(100, s.pct || 0)), lines = s.lines || [];
+      bar.style.width = pct + '%'; bar.className = s.stage === 'failed' ? 'bad' : '';
+      stageEl.textContent = (label[s.stage] || s.stage || 'Waiting') + (pct ? ' · ' + pct + '%' : '');
+      if (lines.length) { const atEnd = logEl.scrollTop + logEl.clientHeight >= logEl.scrollHeight - 4; logEl.textContent = lines.join('\n'); if (atEnd) logEl.scrollTop = logEl.scrollHeight; }
+      if (!s.running && (s.stage === 'done' || s.stage === 'failed')) { again = false; if (onDone) onDone(s); }
+    } catch (e) {
+      misses++; // the server is probably restarting into the new version
+      if (misses > 2) stageEl.textContent = 'Restarting… waiting for it to come back';
+      if (misses > 150) { again = false; stageEl.textContent = 'It has not come back after a few minutes. Check its log on the server.'; }
+    }
+    if (again) setTimeout(tick, 800);
+  };
+  setTimeout(tick, 100);
+  return el;
+}
+
 // compares vX.Y.Z strings: -1 if a is older than b, 1 if newer, 0 if equal or not comparable
 const cmpVer = (a, b) => {
   const p = v => (/^v(\d+)\.(\d+)\.(\d+)$/.exec(v || '') || []).slice(1).map(Number), x = p(a), y = p(b);
@@ -201,7 +233,8 @@ const cmpVer = (a, b) => {
 const latestVersions = async () => { try { return await api('GET', '/api/versions'); } catch (e) { return {}; } };
 // A table cell with a machine's version, a note when a newer release exists, and an Update button
 // when that machine allows being updated from here (it must run with auto-update on).
-const versionCell = (m, latest, pending, updateUrl) => {
+const updateDialog = (m, logUrl) => modal('Updating ' + m.name, h('div', {class:'stack'}, updatePanel(() => api('GET', logUrl), () => refresh()), h('p', {class:'small dim'}, 'You can close this; the update carries on.')));
+const versionCell = (m, latest, pending, updateUrl, logUrl) => {
   const behind = latest && m.version && cmpVer(m.version, latest) < 0, kids = [h('span', {class:'mono small'}, m.version || 'unknown')];
   if (behind) kids.push(h('div', {class:'small'}, latest + ' is available'));
   if (pending) {
@@ -210,7 +243,8 @@ const versionCell = (m, latest, pending, updateUrl) => {
   } else if (m.updateMsg) kids.push(h('div', {class:'small dim'}, m.updateMsg));
   if (behind && !pending) kids.push(h('button', {class:'ghost', disabled:!m.autoUpdate,
     title:m.autoUpdate ? 'Install ' + latest + ' now' : 'Turn on auto-update on that machine (…_AUTO_UPDATE=1) to update it from here',
-    onclick: act(async () => { await api('POST', updateUrl); toast('Update requested'); refresh(); })}, 'Update'));
+    onclick: act(async () => { await api('POST', updateUrl); updateDialog(m, logUrl); refresh(); })}, 'Update'));
+  if (pending || m.updateMsg) kids.push(h('button', {class:'ghost', onclick:() => updateDialog(m, logUrl)}, 'Log'));
   return h('td', {}, h('div', {class:'stack'}, ...kids));
 };
 
@@ -719,7 +753,7 @@ views.sites = async () => {
         h('td', {}, h('span', {class:'dot ' + (st.online[s.id] ? 'ok' : 'bad')})), h('td', {}, s.name),
         h('td', {}, s.local ? 'this machine (direct)' : 'bamboo agent'),
         h('td', {}, s.local ? '—' : ((s.connected || []).map(id => (byId(st.edges, id) || {name:id}).name).join(', ') || 'nothing connected')),
-        s.local ? h('td', {class:'dim'}, '—') : versionCell(s, latest.bamboo, !!s.updateId, O(`/sites/${s.id}/update`)),
+        s.local ? h('td', {class:'dim'}, '—') : versionCell(s, latest.bamboo, !!s.updateId, O(`/sites/${s.id}/update`), O(`/sites/${s.id}/update/log`)),
         h('td', {}, h('button', {class:'ghost', onclick: act(async () => {
           const l = await api('GET', O(`/sites/${s.id}/containers`));
           modal('Containers on ' + s.name, l.length ? h('table', {}, h('thead', {}, h('tr', {}, ['Name', 'Image', 'Ports'].map(t => h('th', {}, t)))), h('tbody', {}, l.map(c => h('tr', {}, h('td', {}, c.name), h('td', {class:'dim'}, c.image), h('td', {class:'mono small'}, c.ports.map(p => (p.public ? p.public + '→' : '') + p.private + '/' + p.type).join(', ')))))) :
@@ -994,7 +1028,7 @@ serverViews.edges = async () => {
   return h('div', {},
     card('Edges', edges.length ? h('table', {}, h('thead', {}, h('tr', {}, ['', 'Name', 'WireGuard endpoint', 'IP addresses', 'Version', 'Orgs', ''].map(t => h('th', {}, t)))),
       h('tbody', {}, edges.map(e => h('tr', {}, h('td', {}, h('span', {class:'dot ' + (e.online ? 'ok' : 'bad'), title:e.online ? 'online' : 'offline'})), h('td', {}, e.name), h('td', {class:'mono'}, e.tunnelAddr),
-        h('td', {class:'mono small'}, (e.ips || []).join(', ') || '—'), versionCell(e, latest.blacktip, e.updatePending, '/api/edges/' + e.id + '/update'), h('td', {class:'small dim'}, (e.orgIds || []).length ? e.orgIds.map(id => (byId(orgs, id) || {name:id}).name).join(', ') : 'all'),
+        h('td', {class:'mono small'}, (e.ips || []).join(', ') || '—'), versionCell(e, latest.blacktip, e.updatePending, '/api/edges/' + e.id + '/update', '/api/edges/' + e.id + '/update/log'), h('td', {class:'small dim'}, (e.orgIds || []).length ? e.orgIds.map(id => (byId(orgs, id) || {name:id}).name).join(', ') : 'all'),
         h('td', {}, h('div', {class:'actions'}, h('button', {class:'ghost', onclick:() => showCmd(e)}, 'Show command'),
           h('button', {class:'ghost danger', onclick: act(async () => { if (confirm('Delete this edge and all its ports?')) { await api('DELETE', '/api/edges/' + e.id); refresh(); } })}, 'Delete'))))))) : h('div', {class:'empty'}, 'None yet.')));
 };
@@ -1063,8 +1097,14 @@ serverViews.settings = async () => {
   const updNow = h('button', {class:'btn', style:'display:none', onclick: act(async () => {
     await api('POST', '/api/admin/update/apply');
     updNow.style.display = 'none';
-    updOut.textContent = 'Updating. Hammerhead restarts in a few seconds; reload this page after that.';
+    showPanel();
   })}, 'Update now');
+  const updBox = h('div', {style:'margin-top:12px'});
+  const showPanel = () => { updOut.textContent = 'Updating Hammerhead. The page reconnects by itself after the restart.'; updBox.replaceChildren(updatePanel(() => api('GET', '/api/admin/update/status'), async s => { if (s.stage === 'done') updOut.textContent = 'Updated: Hammerhead is now running ' + s.version + '.'; })); };
+  api('GET', '/api/admin/update/status').then(s => {
+    if (s.running) showPanel();
+    else if ((s.lines || []).length) updBox.append(h('button', {class:'ghost', onclick:() => { updBox.replaceChildren(updatePanel(() => api('GET', '/api/admin/update/status'))); }}, 'Show the last update log'));
+  }).catch(() => {});
   const checkUpdate = act(async () => {
     updOut.textContent = 'Checking…'; updNow.style.display = 'none';
     const u = await api('GET', '/api/admin/update');
@@ -1081,7 +1121,7 @@ serverViews.settings = async () => {
       h('div', {class:'grid'}, field('SMTP host', f.host), field('Port', f.port), field('Security', f.mode), field('Username', f.user), field('Password', f.pass), field('From', f.from)),
       h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick: act(async () => { await save(); await api('POST', '/api/admin/settings/test-mail'); toast('Test email sent to ' + S.me.email); })}, 'Save and send a test email'))),
     card('Updates', h('p', {class:'dim'}, 'Hammerhead, Blacktip and Bamboo can update themselves from new releases when auto-update is on (HAMMERHEAD_AUTO_UPDATE=1). This checks the newest release for Hammerhead; it installs nothing.'),
-      updOut, h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick:checkUpdate}, 'Check for updates'), updNow)),
+      updOut, h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick:checkUpdate}, 'Check for updates'), updNow), updBox),
     h('button', {class:'btn', onclick:save}, 'Save settings'));
 };