Show edge and site versions with Update buttons; Update now for Hammerhead itself

Edges and sites report their version and whether auto-update is on. Hammerhead can ask a machine that
has auto-update on to install the newest release, and shows the outcome. Requests carry an ID so a
stale result can't answer a new one.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

README.md

@@ -73,6 +73,8 @@ Releases are published for x86-64 only; for ARM machines build it yourself (`TAR
 
 Set `HAMMERHEAD_AUTO_UPDATE=1` (or pass `-auto-update`) and Hammerhead checks this server for a newer release every few hours, verifies it against `SHA256SUMS`, checks that the new binary runs, swaps it in and restarts itself. It only follows full releases (never pre-releases), never downgrades, and keeps the old binary if anything looks wrong. `setup.sh` turns it on. It is not available on Windows, and it is for the systemd and plain-binary installs: with Docker, pull a new image instead.
 
+Edges and sites report their version, so Server → Edges and Sites show which release each one runs and flag a newer one. An **Update** button appears for any machine that runs with auto-update on (`BLACKTIP_AUTO_UPDATE=1` / `BAMBOO_AUTO_UPDATE=1`): it asks that machine to install the newest release now, and the result is shown in the table. Under Server → Settings, **Check for updates** does the same lookup for Hammerhead itself, with an **Update now** button when auto-update is on. A machine without auto-update never installs anything on request.
+
 **With Docker**
 
 ```

agent.go

@@ -76,6 +76,7 @@ func (s *Server) edgeConfig(w http.ResponseWriter, r *http.Request) {
 	if m := s.selfMapping(e); m != nil {
 		cfg.Mappings = append(cfg.Mappings, *m)
 	}
+	cfg.UpdateID = e.UpdateID
 	cfg.MailEgress = []model.MailPolicy{}
 	for _, x := range s.st.MailEgress {
 		if x.EdgeID == e.ID && x.Enabled {
@@ -110,6 +111,7 @@ func (s *Server) edgeReport(w http.ResponseWriter, r *http.Request) {
 	}
 	e.LastSeen = time.Now()
 	e.IPs = in.IPs
+	answered := noteUpdateReport(&e.Version, &e.AutoUpdate, &e.UpdateID, &e.UpdateMsg, in.Version, in.AutoUpdate, in.UpdateID, in.UpdateMsg)
 	for _, x := range s.st.MailEgress {
 		if x.EdgeID == e.ID {
 			if chk, found := in.MailChecks[x.ID]; found {
@@ -146,6 +148,8 @@ func (s *Server) edgeReport(w http.ResponseWriter, r *http.Request) {
 	if e.PublicKey != in.PublicKey {
 		e.PublicKey = in.PublicKey
 		s.bump() // sites need the edge's new key
+	} else if answered {
+		s.bump() // the edge's config no longer carries the update request
 	} else {
 		s.save()
 	}
@@ -205,7 +209,7 @@ func (s *Server) siteConfig(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	st.LastSeen = time.Now()
-	cfg := model.SiteConfig{Version: s.st.Version, SiteID: st.ID, TunnelIP: st.TunnelIP, Edges: []model.EdgeRef{}, MailEgress: []model.MailSpec{}}
+	cfg := model.SiteConfig{Version: s.st.Version, SiteID: st.ID, TunnelIP: st.TunnelIP, Edges: []model.EdgeRef{}, MailEgress: []model.MailSpec{}, UpdateID: st.UpdateID}
 	seen := map[string]bool{}
 	addEdge := func(id string) {
 		if seen[id] {
@@ -249,6 +253,7 @@ func (s *Server) siteReport(w http.ResponseWriter, r *http.Request) {
 		in.Connected = []string{}
 	}
 	st.Connected = in.Connected
+	answered := noteUpdateReport(&st.Version, &st.AutoUpdate, &st.UpdateID, &st.UpdateMsg, in.Version, in.AutoUpdate, in.UpdateID, in.UpdateMsg)
 	s.containers[st.ID] = in.Containers
 	for _, x := range s.st.MailEgress {
 		if x.SiteID != st.ID {
@@ -263,6 +268,8 @@ func (s *Server) siteReport(w http.ResponseWriter, r *http.Request) {
 	if in.PublicKey != "" && st.PublicKey != in.PublicKey {
 		st.PublicKey = in.PublicKey
 		s.bump() // every edge this site talks to needs the new key
+	} else if answered {
+		s.bump() // the site's config no longer carries the update request
 	}
 	ok(w)
 }

e2e/e2e_test.go

@@ -241,6 +241,9 @@ func selfSigned(t *testing.T, host string) *tls.Certificate {
 
 // ---------- the environment ----------
 
+// siteCfgHook lets a test adjust the Bamboo that setup starts.
+var siteCfgHook func(*bamboo.Config)
+
 type world struct {
 	t       *testing.T
 	ts      *httptest.Server
@@ -297,7 +300,11 @@ func setup(t *testing.T, tr blacktip.TraefikConfig, opts ...func(*blacktip.Confi
 		t.Fatal(err)
 	}
 	go bt.Run(ctx)
-	bb, err := bamboo.New(bamboo.Config{Hammerhead: w.ts.URL, Token: site.Token, DataDir: t.TempDir()})
+	bbCfg := bamboo.Config{Hammerhead: w.ts.URL, Token: site.Token, DataDir: t.TempDir()}
+	if siteCfgHook != nil {
+		siteCfgHook(&bbCfg)
+	}
+	bb, err := bamboo.New(bbCfg)
 	if err != nil {
 		t.Fatal(err)
 	}

e2e/updates_test.go

@@ -0,0 +1,106 @@
+package e2e
+
+import (
+	"context"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"git.veillax.com/Veillax/bamboo"
+	"git.veillax.com/Veillax/blacktip"
+	"git.veillax.com/Veillax/reef/model"
+)
+
+type edgeRow struct {
+	ID, Version, UpdateMsg string
+	AutoUpdate             bool
+	UpdatePending          bool
+	Online                 bool
+}
+
+func (w *world) edgeRow() edgeRow {
+	var edges []edgeRow
+	w.admin.must("GET", "/api/edges", nil, &edges)
+	for _, e := range edges {
+		if e.ID == w.edge {
+			return e
+		}
+	}
+	w.t.Fatal("edge not listed")
+	return edgeRow{}
+}
+
+func (w *world) siteRow() model.Site {
+	var st struct{ Sites []model.Site }
+	w.admin.must("GET", "/api/orgs/"+w.org+"/state", nil, &st)
+	for _, s := range st.Sites {
+		if s.ID == w.site {
+			return s
+		}
+	}
+	w.t.Fatal("site not listed")
+	return model.Site{}
+}
+
+func waitFor(t *testing.T, what string, ok func() bool) {
+	t.Helper()
+	for i := 0; i < 150; i++ {
+		if ok() {
+			return
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for %s", what)
+}
+
+func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) {
+	var edgeCalls, siteCalls atomic.Int32
+	siteCfgHook = func(c *bamboo.Config) {
+		c.Version, c.AutoUpdate = "v0.1.0", true
+		c.UpdateNow = func(ctx context.Context) (string, error) { siteCalls.Add(1); return "", nil }
+	}
+	defer func() { siteCfgHook = nil }()
+	w := setup(t, blacktip.TraefikConfig{}, func(c *blacktip.Config) {
+		c.Version, c.AutoUpdate = "v0.1.0", true
+		c.UpdateNow = func(ctx context.Context) (string, error) { edgeCalls.Add(1); return "v0.2.0", nil }
+	})
+
+	waitFor(t, "the edge to report its version", func() bool { e := w.edgeRow(); return e.Version == "v0.1.0" && e.AutoUpdate })
+	waitFor(t, "the site to report its version", func() bool { s := w.siteRow(); return s.Version == "v0.1.0" && s.AutoUpdate })
+
+	w.admin.must("POST", "/api/edges/"+w.edge+"/update", nil, nil)
+	waitFor(t, "the edge to answer the update request", func() bool {
+		e := w.edgeRow()
+		return !e.UpdatePending && e.UpdateMsg == "updated to v0.2.0, restarting"
+	})
+	if edgeCalls.Load() != 1 {
+		t.Fatalf("the edge ran its update %d times, want 1", edgeCalls.Load())
+	}
+
+	w.admin.must("POST", "/api/orgs/"+w.org+"/sites/"+w.site+"/update", nil, nil)
+	waitFor(t, "the site to answer the update request", func() bool {
+		s := w.siteRow()
+		return s.UpdateID == "" && s.UpdateMsg == "already on the newest release"
+	})
+	if siteCalls.Load() != 1 {
+		t.Fatalf("the site ran its update %d times, want 1", siteCalls.Load())
+	}
+
+	// asking again works and is a fresh request, not a leftover answer
+	w.admin.must("POST", "/api/edges/"+w.edge+"/update", nil, nil)
+	waitFor(t, "a second update request to be answered", func() bool { return edgeCalls.Load() == 2 && !w.edgeRow().UpdatePending })
+}
+
+func TestUpdatesNeedAutoUpdateOnTheMachine(t *testing.T) {
+	w := setup(t, blacktip.TraefikConfig{}) // no version, auto-update off
+	waitFor(t, "the edge to be online", func() bool { return w.edgeRow().Online })
+	code, body := w.admin.do("POST", "/api/edges/"+w.edge+"/update", nil, nil)
+	if code != 400 || !strings.Contains(body, "BLACKTIP_AUTO_UPDATE") {
+		t.Fatalf("update on an edge without auto-update: %d %s", code, body)
+	}
+	code, body = w.admin.do("POST", "/api/orgs/"+w.org+"/sites/"+w.local+"/update", nil, nil)
+	if code != 400 {
+		t.Fatalf("update on a local site: %d %s", code, body)
+	}
+}

go.mod

@@ -3,9 +3,9 @@ module git.veillax.com/Veillax/hammerhead
 go 1.27.3
 
 require (
-	git.veillax.com/Veillax/bamboo v0.1.3
+	git.veillax.com/Veillax/bamboo v0.1.4
-	git.veillax.com/Veillax/blacktip v0.1.4
+	git.veillax.com/Veillax/blacktip v0.1.6
-	git.veillax.com/Veillax/reef v0.1.4
+	git.veillax.com/Veillax/reef v0.1.6
 	github.com/emersion/go-msgauth v0.7.0
 	golang.org/x/crypto v0.58.0
 	modernc.org/sqlite v1.61.0

go.sum

@@ -2,16 +2,22 @@ git.veillax.com/Veillax/bamboo v0.1.0 h1:16BG1P8982dIkHA2l0LPS0xYQc8d86PruVAGxc8
 git.veillax.com/Veillax/bamboo v0.1.0/go.mod h1:i4l4uJgG6Ymfwbjnlz0xeTt3qW/+1W1L2JBToMw50n0=
 git.veillax.com/Veillax/bamboo v0.1.3 h1:A66gz9GmmtUq1iIGUAL2ItIeC6iTy5pvqJOoIjRHjFg=
 git.veillax.com/Veillax/bamboo v0.1.3/go.mod h1:UuHV6aNHmV9jpow0xeGlXX0QhMB9glPJUGzgIM7S1MM=
+git.veillax.com/Veillax/bamboo v0.1.4 h1:pgiBJA46CBJx7T3bloCjPekb6wl6W5H8teokFAH5gj0=
+git.veillax.com/Veillax/bamboo v0.1.4/go.mod h1:wzSpfql7U4a3b9mCVH01OPG9H9szrzwSFDufWw02e0g=
 git.veillax.com/Veillax/blacktip v0.1.0 h1:L8WpW56gqtvZ1IAapE95Ynl6NKjqTA98WwmEH01kFFk=
 git.veillax.com/Veillax/blacktip v0.1.0/go.mod h1:sj+mr8393rajzWbl0+RRZrstGyDtZ/GHsut6ebScVxg=
 git.veillax.com/Veillax/blacktip v0.1.4 h1:o3uNoA0ucB/qcN9ccofysdFA6RTEU4jKdgeTWuE2Uw0=
 git.veillax.com/Veillax/blacktip v0.1.4/go.mod h1:f7zXCYlQf/9ffDLjPrMHylKAyXX4Nb+aQ7kErmOOhqI=
+git.veillax.com/Veillax/blacktip v0.1.6 h1:dUJzNGXgieWX05ftWjnMfh6norhaeKTqo1P13lJvjtk=
+git.veillax.com/Veillax/blacktip v0.1.6/go.mod h1:ZnPRc2gPh9ElILfPH39G/x3B6KAiHQyeoDLycSDgVPM=
 git.veillax.com/Veillax/reef v0.1.0 h1:O0OhEKFx2CsvACJ20sfzJNjGsngomtfDdKdxu4ROpCs=
 git.veillax.com/Veillax/reef v0.1.0/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
 git.veillax.com/Veillax/reef v0.1.3 h1:RxSq2xwaup1+MieAkhlvOTZnYwPf19thFHEAe/7VxsI=
 git.veillax.com/Veillax/reef v0.1.3/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
 git.veillax.com/Veillax/reef v0.1.4 h1:T2+B8P1GAitEyTYRkwMT3UHbG9mN5zXb/inE2muXdQ4=
 git.veillax.com/Veillax/reef v0.1.4/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
+git.veillax.com/Veillax/reef v0.1.6 h1:NgZ4u2Z3hJ+BYD/zX9lov8fox4s2i8txtYNOpIsikVo=
+git.veillax.com/Veillax/reef v0.1.6/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso=
 github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g=
 github.com/Microsoft/go-winio v0.6.4/go.mod h1:1xl5JXG4ze/m74GH6Dc4Mwxfc3VL5siRVQjIAlHH07U=
 github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=

infra.go

@@ -24,6 +24,11 @@ type edgeView struct {
 	TunnelAddr string   `json:"tunnelAddr,omitempty"`
 	Token      string   `json:"token,omitempty"`
 	OrgIDs     []string `json:"orgIds,omitempty"`
+
+	Version       string `json:"version,omitempty"`
+	AutoUpdate    bool   `json:"autoUpdate,omitempty"`
+	UpdatePending bool   `json:"updatePending,omitempty"`
+	UpdateMsg     string `json:"updateMsg,omitempty"`
 }
 
 func (s *Server) viewEdge(e *model.Edge, admin bool) edgeView {
@@ -33,6 +38,7 @@ func (s *Server) viewEdge(e *model.Edge, admin bool) edgeView {
 	}
 	if admin {
 		v.TunnelAddr, v.Token, v.OrgIDs = e.TunnelAddr, e.Token, e.OrgIDs
+		v.Version, v.AutoUpdate, v.UpdatePending, v.UpdateMsg = e.Version, e.AutoUpdate, e.UpdateID != "", e.UpdateMsg
 		if v.OrgIDs == nil {
 			v.OrgIDs = []string{}
 		}

server.go

@@ -98,6 +98,10 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("PUT /api/admin/settings", s.withServerAdmin(s.putSettings))
 	mux.HandleFunc("POST /api/admin/settings/test-mail", s.withServerAdmin(s.testMail))
 	mux.HandleFunc("GET /api/admin/update", s.withServerAdmin(s.checkUpdate))
+	mux.HandleFunc("POST /api/admin/update/apply", s.withServerAdmin(s.applyUpdate))
+	mux.HandleFunc("GET /api/versions", s.withUser(s.latestVersions))
+	mux.HandleFunc("POST /api/edges/{id}/update", s.withServerAdmin(s.updateEdge))
+	mux.HandleFunc("POST /api/orgs/{org}/sites/{id}/update", s.withOrg(true, s.updateSite))
 	mux.HandleFunc("GET /api/admin/audit", s.withServerAdmin(func(w http.ResponseWriter, r *http.Request, p *principal) {
 		writeJSON(w, 200, s.logs.queryAudit("", 300, 0))
 	}))

store.go

@@ -208,8 +208,11 @@ type Server struct {
 	limits  map[string][]time.Time
 	oidc    map[string]*oidcDoc
 
-	alerts  map[string]*alertState // active problems by key (see alerts.go)
+	alerts   map[string]*alertState // active problems by key (see alerts.go)
-	mtaSent map[string]time.Time   // reminder throttle for repeat notifications
+	verMu    sync.Mutex             // guards verCache and verAt, the latest releases seen by /api/versions
+	verCache map[string]string
+	verAt    time.Time
+	mtaSent  map[string]time.Time // reminder throttle for repeat notifications
 
 	logs *logStore
 	db   *DB

update.go

@@ -2,30 +2,160 @@ package hammerhead
 
 import (
 	"context"
+	"errors"
 	"net/http"
+	"strings"
+	"sync"
 	"time"
 
 	"git.veillax.com/Veillax/reef/selfupdate"
 )
 
-// checkUpdate asks the release server whether a newer Hammerhead exists. It only looks: updating is
+func (s *Server) updateConfig(app, current string) selfupdate.Config {
-// left to auto-update or to the person running the server.
+	o := s.opts
+	server := o.UpdateServer
+	if server == "" {
+		server = "https://git.veillax.com"
+	}
+	owner := "Veillax"
+	if o.UpdateRepo != "" {
+		owner = strings.SplitN(o.UpdateRepo, "/", 2)[0]
+	}
+	return selfupdate.Config{Server: server, Repo: owner + "/" + app, App: app, Current: current}
+}
+
+// checkUpdate asks the release server whether a newer Hammerhead exists. It only looks.
 func (s *Server) checkUpdate(w http.ResponseWriter, r *http.Request, p *principal) {
 	o := s.opts
-	cfg := selfupdate.Config{Server: o.UpdateServer, Repo: o.UpdateRepo, App: "hammerhead", Current: o.Version}
-	if cfg.Server == "" {
-		cfg.Server = "https://git.veillax.com"
-	}
-	if cfg.Repo == "" {
-		cfg.Repo = "Veillax/hammerhead"
-	}
 	ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
 	defer cancel()
 	out := map[string]any{"current": o.Version, "autoUpdate": o.AutoUpdate, "checked": time.Now()}
-	latest, newer, err := selfupdate.Check(ctx, cfg)
+	latest, newer, err := selfupdate.Check(ctx, s.updateConfig("hammerhead", o.Version))
 	out["latest"], out["newer"] = latest, newer
 	if err != nil {
 		out["error"] = err.Error()
 	}
 	writeJSON(w, 200, out)
 }
+
+// applyUpdate installs the newest release of Hammerhead itself and restarts into it. Like the
+// scheduled check it needs auto-update to be on, which is also what makes the binary replaceable.
+func (s *Server) applyUpdate(w http.ResponseWriter, r *http.Request, p *principal) {
+	if !s.opts.AutoUpdate {
+		fail(w, 400, errors.New("auto-update is off: set HAMMERHEAD_AUTO_UPDATE=1 and restart Hammerhead, or update it by hand"))
+		return
+	}
+	cfg := s.updateConfig("hammerhead", s.opts.Version)
+	cfg.Restart = func(exe string) error { s.Close(); return selfupdate.Restart(exe) }
+	s.audit("", p.User.Email, "server.update", "")
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+		defer cancel()
+		selfupdate.CheckOnce(ctx, cfg) // on success the process is replaced and this never returns
+	}()
+	writeJSON(w, 200, map[string]any{"started": true})
+}
+
+// latestVersions returns the newest release of each program, for "update available" badges. Results
+// are cached for ten minutes so a busy dashboard doesn't hammer the release server.
+func (s *Server) latestVersions(w http.ResponseWriter, r *http.Request, p *principal) {
+	s.verMu.Lock()
+	if time.Since(s.verAt) > 10*time.Minute || s.verCache == nil {
+		ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+		out := map[string]string{}
+		var mu sync.Mutex
+		var wg sync.WaitGroup
+		for _, app := range []string{"hammerhead", "blacktip", "bamboo"} {
+			wg.Add(1)
+			go func(app string) {
+				defer wg.Done()
+				if tag, _, err := selfupdate.Check(ctx, s.updateConfig(app, "")); err == nil && tag != "" {
+					mu.Lock()
+					out[app] = tag
+					mu.Unlock()
+				}
+			}(app)
+		}
+		wg.Wait()
+		cancel()
+		s.verCache, s.verAt = out, time.Now()
+	}
+	out := s.verCache
+	s.verMu.Unlock()
+	writeJSON(w, 200, out)
+}
+
+// noteUpdateReport records what an edge or site says about its version and any update that was
+// requested of it. It reports whether a pending request was answered, which means the config the
+// agent polls has changed (it no longer carries the request).
+func noteUpdateReport(version *string, auto *bool, updateID, updateMsg *string, inVersion string, inAuto bool, inID, inMsg string) bool {
+	upgraded := *version != "" && inVersion != "" && inVersion != *version
+	if inVersion != "" {
+		*version = inVersion
+	}
+	*auto = inAuto
+	if *updateID == "" {
+		return false
+	}
+	switch {
+	case inID == *updateID && inMsg != "":
+		*updateMsg = inMsg
+	case upgraded:
+		*updateMsg = "updated to " + inVersion
+	default:
+		return false
+	}
+	*updateID = ""
+	return true
+}
+
+func (s *Server) updateEdge(w http.ResponseWriter, r *http.Request, p *principal) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	e := s.findEdge(r.PathValue("id"))
+	if e == nil {
+		fail(w, 404, errors.New("no such edge"))
+		return
+	}
+	if err := requestUpdate(&e.AutoUpdate, &e.UpdateID, &e.UpdateMsg, s.edgeOnline(e), "BLACKTIP"); err != nil {
+		fail(w, 400, err)
+		return
+	}
+	s.bump()
+	s.audit("", p.User.Email, "edge.update", e.Name)
+	ok(w)
+}
+
+func (s *Server) updateSite(w http.ResponseWriter, r *http.Request, c *orgCtx) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	st := s.findSite(r.PathValue("id"))
+	if st == nil || !s.siteVisible(c, st) {
+		fail(w, 404, errors.New("no such site"))
+		return
+	}
+	if st.Local {
+		fail(w, 400, errors.New("this site is the edge's own machine; update the edge instead"))
+		return
+	}
+	if err := requestUpdate(&st.AutoUpdate, &st.UpdateID, &st.UpdateMsg, s.siteOnline(st), "BAMBOO"); err != nil {
+		fail(w, 400, err)
+		return
+	}
+	s.bump()
+	s.audit(c.Org.ID, c.Actor, "site.update", st.Name)
+	ok(w)
+}
+
+// requestUpdate marks an update as wanted. The agent installs it only if it has auto-update on, and
+// the request stays until the agent answers (or reports a new version). Asking again replaces it.
+func requestUpdate(auto *bool, updateID, updateMsg *string, online bool, prefix string) error {
+	if !online {
+		return errors.New("it is offline right now")
+	}
+	if !*auto {
+		return errors.New("it isn't running with auto-update on, so it won't install updates from here (set " + prefix + "_AUTO_UPDATE=1 and restart it)")
+	}
+	*updateID, *updateMsg = randHex(8), ""
+	return nil
+}

web/index.html

@@ -184,6 +184,30 @@ const ago = d => { if (!d || d.startsWith('0001')) return 'never'; const s = (Da
 const byId = (list, id) => (list || []).find(x => x.id === id);
 const field = (label, el, hint) => h('label', {}, label, el, hint ? h('span', {class:'sub'}, hint) : null);
 const chk = (label, checked, onchange) => { const i = h('input', {type:'checkbox', checked: !!checked}); if (onchange) i.onchange = () => onchange(i.checked); return Object.assign(h('label', {class:'chk'}, i, label), {box:i}); };
+// ----- versions and updates -----
+// compares vX.Y.Z strings: -1 if a is older than b, 1 if newer, 0 if equal or not comparable
+const cmpVer = (a, b) => {
+  const p = v => (/^v(\d+)\.(\d+)\.(\d+)$/.exec(v || '') || []).slice(1).map(Number), x = p(a), y = p(b);
+  if (x.length < 3 || y.length < 3) return 0;
+  for (let i = 0; i < 3; i++) if (x[i] !== y[i]) return x[i] < y[i] ? -1 : 1;
+  return 0;
+};
+const latestVersions = async () => { try { return await api('GET', '/api/versions'); } catch (e) { return {}; } };
+// A table cell with a machine's version, a note when a newer release exists, and an Update button
+// when that machine allows being updated from here (it must run with auto-update on).
+const versionCell = (m, latest, pending, updateUrl) => {
+  const behind = latest && m.version && cmpVer(m.version, latest) < 0, kids = [h('span', {class:'mono small'}, m.version || 'unknown')];
+  if (behind) kids.push(h('div', {class:'small'}, latest + ' is available'));
+  if (pending) {
+    kids.push(h('div', {class:'small dim'}, 'Updating…'));
+    clearTimeout(S.updTimer); S.updTimer = setTimeout(() => refresh(), 4000); // look again for the result
+  } else if (m.updateMsg) kids.push(h('div', {class:'small dim'}, m.updateMsg));
+  if (behind && !pending) kids.push(h('button', {class:'ghost', disabled:!m.autoUpdate,
+    title:m.autoUpdate ? 'Install ' + latest + ' now' : 'Turn on auto-update on that machine (…_AUTO_UPDATE=1) to update it from here',
+    onclick: act(async () => { await api('POST', updateUrl); toast('Update requested'); refresh(); })}, 'Update'));
+  return h('td', {}, h('div', {class:'stack'}, ...kids));
+};
+
 const card = (title, ...kids) => h('div', {class:'card'}, title ? h('h2', {}, title) : null, ...kids);
 function dialogForm(title, intro, fields, label, submit) {
   const close = modal(title, h('div', {class:'stack'}, intro ? h('p', {class:'dim'}, intro) : null, ...fields,
@@ -581,12 +605,14 @@ views.sites = async () => {
   S.headerAction = ['Add site', () => { const close = modal('Add a site', h('div', {class:'stack'},
     h('p', {class:'dim'}, 'A site is a private network that holds your services. Every edge also has a built-in "this machine" site for services on the same box as Blacktip.'), field('Name', name),
     h('div', {class:'actions'}, h('button', {class:'btn', onclick:() => { close(); add(); }}, 'Add site'), h('button', {class:'ghost', onclick:() => close()}, 'Cancel')))); }];
+  const latest = await latestVersions();
   return h('div', {},
-    card('Sites', h('table', {}, h('thead', {}, h('tr', {}, ['', 'Name', 'Kind', 'Tunnels to', 'Containers', ''].map(t => h('th', {}, t)))),
+    card('Sites', h('table', {}, h('thead', {}, h('tr', {}, ['', 'Name', 'Kind', 'Tunnels to', 'Version', 'Containers', ''].map(t => h('th', {}, t)))),
       h('tbody', {}, st.sites.map(s => h('tr', {},
         h('td', {}, h('span', {class:'dot ' + (st.online[s.id] ? 'ok' : 'bad')})), h('td', {}, s.name),
         h('td', {}, s.local ? 'this machine (direct)' : 'bamboo agent'),
         h('td', {}, s.local ? '—' : ((s.connected || []).map(id => (byId(st.edges, id) || {name:id}).name).join(', ') || 'nothing connected')),
+        s.local ? h('td', {class:'dim'}, '—') : versionCell(s, latest.bamboo, !!s.updateId, O(`/sites/${s.id}/update`)),
         h('td', {}, h('button', {class:'ghost', onclick: act(async () => {
           const l = await api('GET', O(`/sites/${s.id}/containers`));
           modal('Containers on ' + s.name, l.length ? h('table', {}, h('thead', {}, h('tr', {}, ['Name', 'Image', 'Ports'].map(t => h('th', {}, t)))), h('tbody', {}, l.map(c => h('tr', {}, h('td', {}, c.name), h('td', {class:'dim'}, c.image), h('td', {class:'mono small'}, c.ports.map(p => (p.public ? p.public + '→' : '') + p.private + '/' + p.type).join(', ')))))) :
@@ -857,10 +883,11 @@ serverViews.edges = async () => {
     h('div', {class:'actions'}, h('button', {class:'btn', onclick: act(async () => {
       const e = await api('POST', '/api/edges', {name:name.value, tunnelAddr:addr.value, orgIds:orgChecks.filter(x => x.c.box.checked).map(x => x.o.id)}); close(); showCmd(e); refresh(); })}, 'Add edge'),
       h('button', {class:'ghost', onclick:() => close()}, 'Cancel')))); }];
+  const latest = await latestVersions();
   return h('div', {},
-    card('Edges', edges.length ? h('table', {}, h('thead', {}, h('tr', {}, ['', 'Name', 'WireGuard endpoint', 'IP addresses', 'Orgs', ''].map(t => h('th', {}, t)))),
+    card('Edges', edges.length ? h('table', {}, h('thead', {}, h('tr', {}, ['', 'Name', 'WireGuard endpoint', 'IP addresses', 'Version', 'Orgs', ''].map(t => h('th', {}, t)))),
       h('tbody', {}, edges.map(e => h('tr', {}, h('td', {}, h('span', {class:'dot ' + (e.online ? 'ok' : 'bad'), title:e.online ? 'online' : 'offline'})), h('td', {}, e.name), h('td', {class:'mono'}, e.tunnelAddr),
-        h('td', {class:'mono small'}, (e.ips || []).join(', ') || '—'), h('td', {class:'small dim'}, (e.orgIds || []).length ? e.orgIds.map(id => (byId(orgs, id) || {name:id}).name).join(', ') : 'all'),
+        h('td', {class:'mono small'}, (e.ips || []).join(', ') || '—'), versionCell(e, latest.blacktip, e.updatePending, '/api/edges/' + e.id + '/update'), h('td', {class:'small dim'}, (e.orgIds || []).length ? e.orgIds.map(id => (byId(orgs, id) || {name:id}).name).join(', ') : 'all'),
         h('td', {}, h('div', {class:'actions'}, h('button', {class:'ghost', onclick:() => showCmd(e)}, 'Show command'),
           h('button', {class:'ghost danger', onclick: act(async () => { if (confirm('Delete this edge and all its ports?')) { await api('DELETE', '/api/edges/' + e.id); refresh(); } })}, 'Delete'))))))) : h('div', {class:'empty'}, 'None yet.')));
 };
@@ -926,13 +953,18 @@ serverViews.settings = async () => {
   const save = act(async () => { await api('PUT', '/api/admin/settings', {dashboardUrl:f.url.value, acmeEmail:f.acme.value, acmeStaging:staging.box.checked,
     smtp:{host:f.host.value, port:+f.port.value, user:f.user.value, pass:f.pass.value, from:f.from.value, mode:f.mode.value}}); S.me = await api('GET', '/api/me'); toast('Saved'); });
   const updOut = h('div', {class:'dim'}, 'Not checked yet.');
+  const updNow = h('button', {class:'btn', style:'display:none', onclick: act(async () => {
+    await api('POST', '/api/admin/update/apply');
+    updNow.style.display = 'none';
+    updOut.textContent = 'Updating. Hammerhead restarts in a few seconds; reload this page after that.';
+  })}, 'Update now');
   const checkUpdate = act(async () => {
-    updOut.textContent = 'Checking…';
+    updOut.textContent = 'Checking…'; updNow.style.display = 'none';
     const u = await api('GET', '/api/admin/update');
     const dev = !/^v\d+\.\d+\.\d+$/.test(u.current || '');
     if (u.error) updOut.textContent = 'Running ' + (u.current || 'an unknown version') + '. Could not check for updates: ' + u.error;
     else if (dev) updOut.textContent = 'This is a development build (' + (u.current || 'unversioned') + '). The newest release is ' + u.latest + '.';
-    else if (u.newer) updOut.textContent = 'Running ' + u.current + '. ' + u.latest + ' is available. ' + (u.autoUpdate ? 'Auto-update is on, so it installs by itself within about six hours.' : 'Auto-update is off: install it yourself, or set HAMMERHEAD_AUTO_UPDATE=1 and restart.');
+    else if (u.newer) { updOut.textContent = 'Running ' + u.current + '. ' + u.latest + ' is available. ' + (u.autoUpdate ? 'Auto-update is on, so it installs by itself within about six hours, or you can install it now.' : 'Auto-update is off: install it yourself, or set HAMMERHEAD_AUTO_UPDATE=1 and restart.'); if (u.autoUpdate) updNow.style.display = ''; }
     else updOut.textContent = 'Running ' + u.current + ', which is the newest release.';
   });
   return h('div', {},
@@ -942,7 +974,7 @@ serverViews.settings = async () => {
       h('div', {class:'grid'}, field('SMTP host', f.host), field('Port', f.port), field('Security', f.mode), field('Username', f.user), field('Password', f.pass), field('From', f.from)),
       h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick: act(async () => { await save(); await api('POST', '/api/admin/settings/test-mail'); toast('Test email sent to ' + S.me.email); })}, 'Save and send a test email'))),
     card('Updates', h('p', {class:'dim'}, 'Hammerhead, Blacktip and Bamboo can update themselves from new releases when auto-update is on (HAMMERHEAD_AUTO_UPDATE=1). This checks the newest release for Hammerhead; it installs nothing.'),
-      updOut, h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick:checkUpdate}, 'Check for updates'))),
+      updOut, h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick:checkUpdate}, 'Check for updates'), updNow)),
     h('button', {class:'btn', onclick:save}, 'Save settings'));
 };