From 2b7cb5dd358c02e505451204f0dfd42c51977804 Mon Sep 17 00:00:00 2001 From: Veillax <126523099+Veillax@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:05:42 -0600 Subject: [PATCH] Show a progress bar and live log while Hammerhead, edges and sites update The updater reports each step (checking, downloading with percent, verifying, installing, restarting). Edges and sites send that progress and their log with their reports; Hammerhead serves it to a dialog that opens when an update starts, and keeps its own last update log on disk across the restart. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 2 +- agent.go | 4 ++ e2e/updates_test.go | 30 ++++++++- go.mod | 6 +- go.sum | 6 ++ server.go | 3 + store.go | 4 ++ update.go | 24 ++++++- update_test.go | 36 +++++++++++ updatelog.go | 152 ++++++++++++++++++++++++++++++++++++++++++++ web/index.html | 52 +++++++++++++-- 11 files changed, 306 insertions(+), 13 deletions(-) create mode 100644 updatelog.go diff --git a/README.md b/README.md index 0fbf7d6..93a4670 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Releases are published for x86-64 only; for ARM machines build it yourself (`TAR Set `HAMMERHEAD_AUTO_UPDATE=1` (or pass `-auto-update`) and Hammerhead checks this server for a newer release every few hours, verifies it against `SHA256SUMS`, checks that the new binary runs, swaps it in and restarts itself. It only follows full releases (never pre-releases), never downgrades, and keeps the old binary if anything looks wrong. `setup.sh` turns it on. It is not available on Windows, and it is for the systemd and plain-binary installs: with Docker, pull a new image instead. -Edges and sites report their version, so Server → Edges and Sites show which release each one runs and flag a newer one. An **Update** button appears for any machine that runs with auto-update on (`BLACKTIP_AUTO_UPDATE=1` / `BAMBOO_AUTO_UPDATE=1`): it asks that machine to install the newest release now, and the result is shown in the table. Under Server → Settings, **Check for updates** does the same lookup for Hammerhead itself, with an **Update now** button when auto-update is on. A machine without auto-update never installs anything on request. +Edges and sites report their version, so Server → Edges and Sites show which release each one runs and flag a newer one. An **Update** button appears for any machine that runs with auto-update on (`BLACKTIP_AUTO_UPDATE=1` / `BAMBOO_AUTO_UPDATE=1`): it asks that machine to install the newest release now, and the result is shown in the table. Under Server → Settings, **Check for updates** does the same lookup for Hammerhead itself, with an **Update now** button when auto-update is on. A machine without auto-update never installs anything on request. Starting an update opens a progress bar with the live log of what the machine is doing (checking, downloading, verifying, installing, restarting). Hammerhead keeps the log of its own last update in `update.log` in its data folder, so it is still there after the restart. **With Docker** diff --git a/agent.go b/agent.go index 23abfe0..b62b621 100644 --- a/agent.go +++ b/agent.go @@ -111,7 +111,9 @@ func (s *Server) edgeReport(w http.ResponseWriter, r *http.Request) { } e.LastSeen = time.Now() e.IPs = in.IPs + oldVersion := e.Version answered := noteUpdateReport(&e.Version, &e.AutoUpdate, &e.UpdateID, &e.UpdateMsg, in.Version, in.AutoUpdate, in.UpdateID, in.UpdateMsg) + s.recordUpdateProgress(e.ID, in.UpdateID, in.UpdateStage, in.UpdatePct, in.UpdateLog, oldVersion, e.Version) for _, x := range s.st.MailEgress { if x.EdgeID == e.ID { if chk, found := in.MailChecks[x.ID]; found { @@ -253,7 +255,9 @@ func (s *Server) siteReport(w http.ResponseWriter, r *http.Request) { in.Connected = []string{} } st.Connected = in.Connected + oldVersion := st.Version answered := noteUpdateReport(&st.Version, &st.AutoUpdate, &st.UpdateID, &st.UpdateMsg, in.Version, in.AutoUpdate, in.UpdateID, in.UpdateMsg) + s.recordUpdateProgress(st.ID, in.UpdateID, in.UpdateStage, in.UpdatePct, in.UpdateLog, oldVersion, st.Version) s.containers[st.ID] = in.Containers for _, x := range s.st.MailEgress { if x.SiteID != st.ID { diff --git a/e2e/updates_test.go b/e2e/updates_test.go index 70ed115..fd34956 100644 --- a/e2e/updates_test.go +++ b/e2e/updates_test.go @@ -58,12 +58,23 @@ func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) { var edgeCalls, siteCalls atomic.Int32 siteCfgHook = func(c *bamboo.Config) { c.Version, c.AutoUpdate = "v0.1.0", true - c.UpdateNow = func(ctx context.Context) (string, error) { siteCalls.Add(1); return "", nil } + c.UpdateNow = func(ctx context.Context, progress func(string, int, string)) (string, error) { + siteCalls.Add(1) + progress("checking", 10, "Checking for the newest release") + progress("done", 100, "Already on the newest release") + return "", nil + } } defer func() { siteCfgHook = nil }() w := setup(t, blacktip.TraefikConfig{}, func(c *blacktip.Config) { c.Version, c.AutoUpdate = "v0.1.0", true - c.UpdateNow = func(ctx context.Context) (string, error) { edgeCalls.Add(1); return "v0.2.0", nil } + c.UpdateNow = func(ctx context.Context, progress func(string, int, string)) (string, error) { + edgeCalls.Add(1) + progress("downloading", 40, "Downloading blacktip_v0.2.0_linux_amd64.tar.gz") + progress("downloading", 70, "Downloaded 28.0 of 40.0 MB (70%)") + progress("restarting", 100, "Installed v0.2.0; restarting into it") + return "v0.2.0", nil + } }) waitFor(t, "the edge to report its version", func() bool { e := w.edgeRow(); return e.Version == "v0.1.0" && e.AutoUpdate }) @@ -77,6 +88,17 @@ func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) { if edgeCalls.Load() != 1 { t.Fatalf("the edge ran its update %d times, want 1", edgeCalls.Load()) } + var log struct { + Running bool + Stage string + Pct int + Lines []string + } + w.admin.must("GET", "/api/edges/"+w.edge+"/update/log", nil, &log) + text := strings.Join(log.Lines, "; ") + if log.Running || log.Stage != "restarting" || log.Pct != 100 || !strings.Contains(text, "Update requested") || !strings.Contains(text, "Downloaded 28.0 of 40.0 MB") || !strings.Contains(text, "restarting into it") { + t.Fatalf("edge update log: %+v (%s)", log, text) + } w.admin.must("POST", "/api/orgs/"+w.org+"/sites/"+w.site+"/update", nil, nil) waitFor(t, "the site to answer the update request", func() bool { @@ -86,6 +108,10 @@ func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) { if siteCalls.Load() != 1 { t.Fatalf("the site ran its update %d times, want 1", siteCalls.Load()) } + w.admin.must("GET", "/api/orgs/"+w.org+"/sites/"+w.site+"/update/log", nil, &log) + if log.Running || log.Stage != "done" || !strings.Contains(strings.Join(log.Lines, "; "), "Already on the newest release") { + t.Fatalf("site update log: %+v", log) + } // asking again works and is a fresh request, not a leftover answer w.admin.must("POST", "/api/edges/"+w.edge+"/update", nil, nil) diff --git a/go.mod b/go.mod index 9d7f594..9a19468 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.veillax.com/Veillax/hammerhead go 1.27.3 require ( - git.veillax.com/Veillax/bamboo v0.1.4 - git.veillax.com/Veillax/blacktip v0.1.6 - git.veillax.com/Veillax/reef v0.1.6 + git.veillax.com/Veillax/bamboo v0.1.5 + git.veillax.com/Veillax/blacktip v0.1.7 + git.veillax.com/Veillax/reef v0.1.7 github.com/emersion/go-msgauth v0.7.0 golang.org/x/crypto v0.58.0 modernc.org/sqlite v1.61.0 diff --git a/go.sum b/go.sum index 4db7313..c194553 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,16 @@ git.veillax.com/Veillax/bamboo v0.1.3 h1:A66gz9GmmtUq1iIGUAL2ItIeC6iTy5pvqJOoIjR git.veillax.com/Veillax/bamboo v0.1.3/go.mod h1:UuHV6aNHmV9jpow0xeGlXX0QhMB9glPJUGzgIM7S1MM= git.veillax.com/Veillax/bamboo v0.1.4 h1:pgiBJA46CBJx7T3bloCjPekb6wl6W5H8teokFAH5gj0= git.veillax.com/Veillax/bamboo v0.1.4/go.mod h1:wzSpfql7U4a3b9mCVH01OPG9H9szrzwSFDufWw02e0g= +git.veillax.com/Veillax/bamboo v0.1.5 h1:i81c75/7wjayP9tOQ3tSnJWTBhvbjAwNBEPhT+3P2Xk= +git.veillax.com/Veillax/bamboo v0.1.5/go.mod h1:se4kkcP6c3uCGD4HQd+nDLoblS0lN8qa+eFWLo5Horw= git.veillax.com/Veillax/blacktip v0.1.0 h1:L8WpW56gqtvZ1IAapE95Ynl6NKjqTA98WwmEH01kFFk= git.veillax.com/Veillax/blacktip v0.1.0/go.mod h1:sj+mr8393rajzWbl0+RRZrstGyDtZ/GHsut6ebScVxg= git.veillax.com/Veillax/blacktip v0.1.4 h1:o3uNoA0ucB/qcN9ccofysdFA6RTEU4jKdgeTWuE2Uw0= git.veillax.com/Veillax/blacktip v0.1.4/go.mod h1:f7zXCYlQf/9ffDLjPrMHylKAyXX4Nb+aQ7kErmOOhqI= git.veillax.com/Veillax/blacktip v0.1.6 h1:dUJzNGXgieWX05ftWjnMfh6norhaeKTqo1P13lJvjtk= git.veillax.com/Veillax/blacktip v0.1.6/go.mod h1:ZnPRc2gPh9ElILfPH39G/x3B6KAiHQyeoDLycSDgVPM= +git.veillax.com/Veillax/blacktip v0.1.7 h1:PjxEkhE4fnGWht9Pj2/l6S8fOX2Iqjir3rc9vfpMFiQ= +git.veillax.com/Veillax/blacktip v0.1.7/go.mod h1:dVm4wC6wufPjFzK8pvHnqNJJlxroRYmnq8mdkCqOKGc= git.veillax.com/Veillax/reef v0.1.0 h1:O0OhEKFx2CsvACJ20sfzJNjGsngomtfDdKdxu4ROpCs= git.veillax.com/Veillax/reef v0.1.0/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso= git.veillax.com/Veillax/reef v0.1.3 h1:RxSq2xwaup1+MieAkhlvOTZnYwPf19thFHEAe/7VxsI= @@ -18,6 +22,8 @@ git.veillax.com/Veillax/reef v0.1.4 h1:T2+B8P1GAitEyTYRkwMT3UHbG9mN5zXb/inE2muXd git.veillax.com/Veillax/reef v0.1.4/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso= git.veillax.com/Veillax/reef v0.1.6 h1:NgZ4u2Z3hJ+BYD/zX9lov8fox4s2i8txtYNOpIsikVo= git.veillax.com/Veillax/reef v0.1.6/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso= +git.veillax.com/Veillax/reef v0.1.7 h1:5C2udonlJac2xvDOE5e+zDxej+AbZ9ttrNSE4owmwuU= +git.veillax.com/Veillax/reef v0.1.7/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso= github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g= github.com/Microsoft/go-winio v0.6.4/go.mod h1:1xl5JXG4ze/m74GH6Dc4Mwxfc3VL5siRVQjIAlHH07U= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= diff --git a/server.go b/server.go index 00866e6..661c99f 100644 --- a/server.go +++ b/server.go @@ -99,7 +99,10 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /api/admin/settings/test-mail", s.withServerAdmin(s.testMail)) mux.HandleFunc("GET /api/admin/update", s.withServerAdmin(s.checkUpdate)) mux.HandleFunc("POST /api/admin/update/apply", s.withServerAdmin(s.applyUpdate)) + mux.HandleFunc("GET /api/admin/update/status", s.withServerAdmin(s.updateStatus)) mux.HandleFunc("GET /api/versions", s.withUser(s.latestVersions)) + mux.HandleFunc("GET /api/edges/{id}/update/log", s.withServerAdmin(s.edgeUpdateLog)) + mux.HandleFunc("GET /api/orgs/{org}/sites/{id}/update/log", s.withOrg(true, s.siteUpdateLog)) mux.HandleFunc("POST /api/edges/{id}/update", s.withServerAdmin(s.updateEdge)) mux.HandleFunc("POST /api/orgs/{org}/sites/{id}/update", s.withOrg(true, s.updateSite)) mux.HandleFunc("GET /api/admin/audit", s.withServerAdmin(func(w http.ResponseWriter, r *http.Request, p *principal) { diff --git a/store.go b/store.go index 62de4c9..6a08c15 100644 --- a/store.go +++ b/store.go @@ -209,6 +209,8 @@ type Server struct { oidc map[string]*oidcDoc alerts map[string]*alertState // active problems by key (see alerts.go) + updRuns map[string]*updRun // progress of updates running on edges and sites, by ID + selfUpd selfUpdate // progress of this server's own update verMu sync.Mutex // guards verCache and verAt, the latest releases seen by /api/versions verCache map[string]string verAt time.Time @@ -253,6 +255,7 @@ func New(dir string, opts Options) (*Server, error) { oidc: map[string]*oidcDoc{}, alerts: map[string]*alertState{}, mtaSent: map[string]time.Time{}, + updRuns: map[string]*updRun{}, } s.st.Sessions = map[string]*Session{} s.st.Secrets = map[string]*Secret{} @@ -282,6 +285,7 @@ func New(dir string, opts Options) (*Server, error) { if opts.SelfDomain != "" && opts.SelfTunnel == "" { s.opts.SelfTunnel = opts.SelfDomain + ":51820" } + s.finishSelfUpdateLog() if err := s.ensureSelfEdge(); err != nil { db.Close() return nil, err diff --git a/update.go b/update.go index d9e1743..393e95f 100644 --- a/update.go +++ b/update.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "os" "strings" "sync" "time" @@ -45,13 +46,32 @@ func (s *Server) applyUpdate(w http.ResponseWriter, r *http.Request, p *principa fail(w, 400, errors.New("auto-update is off: set HAMMERHEAD_AUTO_UPDATE=1 and restart Hammerhead, or update it by hand")) return } + u := &s.selfUpd + u.mu.Lock() + if u.running { + u.mu.Unlock() + fail(w, 409, errors.New("an update is already running")) + return + } + u.running, u.stage, u.pct, u.lines = true, "starting", 0, nil + u.mu.Unlock() + os.WriteFile(s.updateLogPath(), nil, 0o600) // a new run starts a new log cfg := s.updateConfig("hammerhead", s.opts.Version) - cfg.Restart = func(exe string) error { s.Close(); return selfupdate.Restart(exe) } + cfg.Progress = func(e selfupdate.Event) { s.selfNote(e.Stage, e.Pct, e.Line) } + cfg.Restart = func(exe string) error { + s.selfNote("restarting", 100, "Closing the database and restarting") + s.Close() + return selfupdate.Restart(exe) + } + s.selfNote("starting", 0, "Update requested by "+p.User.Email) s.audit("", p.User.Email, "server.update", "") go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() selfupdate.CheckOnce(ctx, cfg) // on success the process is replaced and this never returns + u.mu.Lock() + u.running = false + u.mu.Unlock() }() writeJSON(w, 200, map[string]any{"started": true}) } @@ -121,6 +141,7 @@ func (s *Server) updateEdge(w http.ResponseWriter, r *http.Request, p *principal fail(w, 400, err) return } + delete(s.updRuns, e.ID) // a new request starts a new log s.bump() s.audit("", p.User.Email, "edge.update", e.Name) ok(w) @@ -142,6 +163,7 @@ func (s *Server) updateSite(w http.ResponseWriter, r *http.Request, c *orgCtx) { fail(w, 400, err) return } + delete(s.updRuns, st.ID) s.bump() s.audit(c.Org.ID, c.Actor, "site.update", st.Name) ok(w) diff --git a/update_test.go b/update_test.go index 1057df9..aa63d43 100644 --- a/update_test.go +++ b/update_test.go @@ -5,6 +5,9 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" ) @@ -52,3 +55,36 @@ func TestCheckForUpdates(t *testing.T) { t.Errorf("unreachable server: %d %s", code, b) } } + +func TestSelfUpdateLogSurvivesTheRestartItEndsWith(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "update.log"), []byte("12:00:00 Checking\n12:00:09 Installed v0.3.0; restarting into it\n"), 0o600); err != nil { + t.Fatal(err) + } + s, err := New(dir, Options{AdminEmail: "admin@test.local", AdminPassword: "password123", MasterKey: key, Version: "v0.3.0"}) + if err != nil { + t.Fatal(err) + } + defer s.Close() + do := apiClient(t, s) + code, b := do("GET", "/api/admin/update/status", nil) + var out struct { + Running bool + Stage string + Pct int + Lines []string + Version string + } + json.Unmarshal(b, &out) + last := "" + if len(out.Lines) > 0 { + last = out.Lines[len(out.Lines)-1] + } + if code != 200 || out.Running || out.Stage != "done" || out.Pct != 100 || len(out.Lines) != 3 || !strings.Contains(last, "now running v0.3.0") { + t.Fatalf("status: %d %s", code, b) + } + // starting an update needs auto-update on + if code, b := do("POST", "/api/admin/update/apply", nil); code != 400 || !strings.Contains(string(b), "HAMMERHEAD_AUTO_UPDATE") { + t.Fatalf("apply without auto-update: %d %s", code, b) + } +} diff --git a/updatelog.go b/updatelog.go new file mode 100644 index 0000000..440eddc --- /dev/null +++ b/updatelog.go @@ -0,0 +1,152 @@ +package hammerhead + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const maxUpdateLines = 200 + +// updRun is the progress of one update an edge or site is carrying out, as that agent reported it. +type updRun struct { + ID string // the update request it belongs to + Stage string + Pct int + Lines []string +} + +// recordUpdateProgress keeps what an agent says about the update it is running. When the agent +// reports a new version, the update is finished: its log gets a closing line and is then left alone, +// so the freshly restarted program (which starts a new, empty log) can't overwrite it. +// Caller holds s.mu. +func (s *Server) recordUpdateProgress(agentID, reqID, stage string, pct int, lines []string, oldVersion, newVersion string) { + run := s.updRuns[agentID] + if run != nil && run.Stage != "done" && oldVersion != "" && newVersion != "" && newVersion != oldVersion { + run.Lines = append(run.Lines, time.Now().Format("15:04:05")+" Now running "+newVersion) + run.Stage, run.Pct = "done", 100 + return + } + if reqID == "" || (stage == "" && len(lines) == 0) { + return + } + if run != nil && run.ID == reqID && run.Stage == "done" { + return + } + s.updRuns[agentID] = &updRun{ID: reqID, Stage: stage, Pct: pct, Lines: lines} +} + +func (s *Server) updateLogView(agentID string, pending bool, msg, version string) map[string]any { + out := map[string]any{"running": pending, "msg": msg, "version": version, "stage": "", "pct": 0, "lines": []string{}} + if run := s.updRuns[agentID]; run != nil { + out["stage"], out["pct"], out["lines"] = run.Stage, run.Pct, run.Lines + } + return out +} + +func (s *Server) edgeUpdateLog(w http.ResponseWriter, r *http.Request, p *principal) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.findEdge(r.PathValue("id")) + if e == nil { + fail(w, 404, errors.New("no such edge")) + return + } + writeJSON(w, 200, s.updateLogView(e.ID, e.UpdateID != "", e.UpdateMsg, e.Version)) +} + +func (s *Server) siteUpdateLog(w http.ResponseWriter, r *http.Request, c *orgCtx) { + s.mu.Lock() + defer s.mu.Unlock() + st := s.findSite(r.PathValue("id")) + if st == nil || !s.siteVisible(c, st) { + fail(w, 404, errors.New("no such site")) + return + } + writeJSON(w, 200, s.updateLogView(st.ID, st.UpdateID != "", st.UpdateMsg, st.Version)) +} + +// selfUpdate is the progress of this server's own update. It is kept in memory and in /update.log, +// so the page can still show it after the restart that ends a successful update. +type selfUpdate struct { + mu sync.Mutex + running bool + stage string + pct int + lines []string +} + +func (s *Server) updateLogPath() string { return filepath.Join(s.dir, "update.log") } + +func (s *Server) selfNote(stage string, pct int, line string) { + u := &s.selfUpd + u.mu.Lock() + defer u.mu.Unlock() + if stage != "" { + u.stage = stage + } + if pct >= 0 { + u.pct = pct + } + if line == "" { + return + } + line = time.Now().Format("15:04:05") + " " + line + u.lines = append(u.lines, line) + if len(u.lines) > maxUpdateLines { + u.lines = u.lines[len(u.lines)-maxUpdateLines:] + } + if f, err := os.OpenFile(s.updateLogPath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600); err == nil { + f.WriteString(line + "\n") + f.Close() + } +} + +func readLines(path string) []string { + b, err := os.ReadFile(path) + if err != nil { + return nil + } + lines := strings.Split(strings.TrimRight(string(b), "\n"), "\n") + if len(lines) == 1 && lines[0] == "" { + return nil + } + if len(lines) > maxUpdateLines { + lines = lines[len(lines)-maxUpdateLines:] + } + return lines +} + +// finishSelfUpdateLog runs at startup: if the last run ended by restarting into this process, say so. +func (s *Server) finishSelfUpdateLog() { + lines := readLines(s.updateLogPath()) + if n := len(lines); n > 0 && strings.Contains(lines[n-1], "restarting") && s.opts.Version != "" { + if f, err := os.OpenFile(s.updateLogPath(), os.O_APPEND|os.O_WRONLY, 0o600); err == nil { + f.WriteString(time.Now().Format("15:04:05") + " Hammerhead restarted: now running " + s.opts.Version + "\n") + f.Close() + } + } +} + +// updateStatus returns the progress and log of this server's own update, for the Settings page. +func (s *Server) updateStatus(w http.ResponseWriter, r *http.Request, p *principal) { + u := &s.selfUpd + u.mu.Lock() + out := map[string]any{"running": u.running, "stage": u.stage, "pct": u.pct, "lines": append([]string(nil), u.lines...), "version": s.opts.Version} + u.mu.Unlock() + if lines, _ := out["lines"].([]string); len(lines) == 0 { + // nothing in memory: this is a fresh process, so show the last update's log from disk + if saved := readLines(s.updateLogPath()); len(saved) > 0 { + stage := "done" + if strings.Contains(saved[len(saved)-1], "Failed:") { + stage = "failed" + } + out["lines"], out["stage"], out["pct"] = saved, stage, 100 + } + } + writeJSON(w, 200, out) +} diff --git a/web/index.html b/web/index.html index 5f3607c..5080ea8 100644 --- a/web/index.html +++ b/web/index.html @@ -76,6 +76,10 @@ .choice.on { border-color:var(--accent); background:var(--accent-soft); } .choice-t { font-weight:600; } .sect-h { padding:var(--s3) var(--s4); font-weight:600; border-bottom:1px solid var(--border); } + .progress { height:8px; background:var(--surface-2); border-radius:999px; overflow:hidden; } + .progress > div { height:100%; background:var(--accent); transition:width .4s ease; } + .progress > div.bad { background:var(--danger); } + .logbox { margin:0; background:var(--bg); border:1px solid var(--border); border-radius:var(--radius); padding:var(--s3); font:var(--text-xs)/1.6 var(--font-mono); max-height:260px; overflow:auto; white-space:pre-wrap; word-break:break-word; } /* forms */ label { display:flex; flex-direction:column; gap:var(--s1); font-size:var(--text-sm); font-weight:600; color:var(--text); } @@ -191,6 +195,34 @@ const byId = (list, id) => (list || []).find(x => x.id === id); const field = (label, el, hint) => h('label', {}, label, el, hint ? h('span', {class:'sub'}, hint) : null); const chk = (label, checked, onchange) => { const i = h('input', {type:'checkbox', checked: !!checked}); if (onchange) i.onchange = () => onchange(i.checked); return Object.assign(h('label', {class:'chk'}, i, label), {box:i}); }; // ----- versions and updates ----- +// A progress bar and live log for an update. load() returns {running, stage, pct, lines}; the panel polls it +// until the update is over, and keeps going through the gap while a program restarts into its new version. +function updatePanel(load, onDone) { + const bar = h('div', {style:'width:0%'}), stageEl = h('div', {class:'small'}, 'Starting…'), logEl = h('pre', {class:'logbox'}, 'Waiting for the first message…'); + const el = h('div', {class:'stack'}, stageEl, h('div', {class:'progress'}, bar), logEl); + const label = {starting:'Starting', checking:'Checking for the release', downloading:'Downloading', verifying:'Verifying', installing:'Installing', restarting:'Restarting', done:'Done', failed:'Failed'}; + let misses = 0; + const tick = async () => { + if (!el.isConnected) return; // the dialog or card was closed + let again = true; + try { + const s = await load(); misses = 0; + const pct = Math.max(0, Math.min(100, s.pct || 0)), lines = s.lines || []; + bar.style.width = pct + '%'; bar.className = s.stage === 'failed' ? 'bad' : ''; + stageEl.textContent = (label[s.stage] || s.stage || 'Waiting') + (pct ? ' · ' + pct + '%' : ''); + if (lines.length) { const atEnd = logEl.scrollTop + logEl.clientHeight >= logEl.scrollHeight - 4; logEl.textContent = lines.join('\n'); if (atEnd) logEl.scrollTop = logEl.scrollHeight; } + if (!s.running && (s.stage === 'done' || s.stage === 'failed')) { again = false; if (onDone) onDone(s); } + } catch (e) { + misses++; // the server is probably restarting into the new version + if (misses > 2) stageEl.textContent = 'Restarting… waiting for it to come back'; + if (misses > 150) { again = false; stageEl.textContent = 'It has not come back after a few minutes. Check its log on the server.'; } + } + if (again) setTimeout(tick, 800); + }; + setTimeout(tick, 100); + return el; +} + // compares vX.Y.Z strings: -1 if a is older than b, 1 if newer, 0 if equal or not comparable const cmpVer = (a, b) => { const p = v => (/^v(\d+)\.(\d+)\.(\d+)$/.exec(v || '') || []).slice(1).map(Number), x = p(a), y = p(b); @@ -201,7 +233,8 @@ const cmpVer = (a, b) => { const latestVersions = async () => { try { return await api('GET', '/api/versions'); } catch (e) { return {}; } }; // A table cell with a machine's version, a note when a newer release exists, and an Update button // when that machine allows being updated from here (it must run with auto-update on). -const versionCell = (m, latest, pending, updateUrl) => { +const updateDialog = (m, logUrl) => modal('Updating ' + m.name, h('div', {class:'stack'}, updatePanel(() => api('GET', logUrl), () => refresh()), h('p', {class:'small dim'}, 'You can close this; the update carries on.'))); +const versionCell = (m, latest, pending, updateUrl, logUrl) => { const behind = latest && m.version && cmpVer(m.version, latest) < 0, kids = [h('span', {class:'mono small'}, m.version || 'unknown')]; if (behind) kids.push(h('div', {class:'small'}, latest + ' is available')); if (pending) { @@ -210,7 +243,8 @@ const versionCell = (m, latest, pending, updateUrl) => { } else if (m.updateMsg) kids.push(h('div', {class:'small dim'}, m.updateMsg)); if (behind && !pending) kids.push(h('button', {class:'ghost', disabled:!m.autoUpdate, title:m.autoUpdate ? 'Install ' + latest + ' now' : 'Turn on auto-update on that machine (…_AUTO_UPDATE=1) to update it from here', - onclick: act(async () => { await api('POST', updateUrl); toast('Update requested'); refresh(); })}, 'Update')); + onclick: act(async () => { await api('POST', updateUrl); updateDialog(m, logUrl); refresh(); })}, 'Update')); + if (pending || m.updateMsg) kids.push(h('button', {class:'ghost', onclick:() => updateDialog(m, logUrl)}, 'Log')); return h('td', {}, h('div', {class:'stack'}, ...kids)); }; @@ -719,7 +753,7 @@ views.sites = async () => { h('td', {}, h('span', {class:'dot ' + (st.online[s.id] ? 'ok' : 'bad')})), h('td', {}, s.name), h('td', {}, s.local ? 'this machine (direct)' : 'bamboo agent'), h('td', {}, s.local ? '—' : ((s.connected || []).map(id => (byId(st.edges, id) || {name:id}).name).join(', ') || 'nothing connected')), - s.local ? h('td', {class:'dim'}, '—') : versionCell(s, latest.bamboo, !!s.updateId, O(`/sites/${s.id}/update`)), + s.local ? h('td', {class:'dim'}, '—') : versionCell(s, latest.bamboo, !!s.updateId, O(`/sites/${s.id}/update`), O(`/sites/${s.id}/update/log`)), h('td', {}, h('button', {class:'ghost', onclick: act(async () => { const l = await api('GET', O(`/sites/${s.id}/containers`)); modal('Containers on ' + s.name, l.length ? h('table', {}, h('thead', {}, h('tr', {}, ['Name', 'Image', 'Ports'].map(t => h('th', {}, t)))), h('tbody', {}, l.map(c => h('tr', {}, h('td', {}, c.name), h('td', {class:'dim'}, c.image), h('td', {class:'mono small'}, c.ports.map(p => (p.public ? p.public + '→' : '') + p.private + '/' + p.type).join(', ')))))) : @@ -994,7 +1028,7 @@ serverViews.edges = async () => { return h('div', {}, card('Edges', edges.length ? h('table', {}, h('thead', {}, h('tr', {}, ['', 'Name', 'WireGuard endpoint', 'IP addresses', 'Version', 'Orgs', ''].map(t => h('th', {}, t)))), h('tbody', {}, edges.map(e => h('tr', {}, h('td', {}, h('span', {class:'dot ' + (e.online ? 'ok' : 'bad'), title:e.online ? 'online' : 'offline'})), h('td', {}, e.name), h('td', {class:'mono'}, e.tunnelAddr), - h('td', {class:'mono small'}, (e.ips || []).join(', ') || '—'), versionCell(e, latest.blacktip, e.updatePending, '/api/edges/' + e.id + '/update'), h('td', {class:'small dim'}, (e.orgIds || []).length ? e.orgIds.map(id => (byId(orgs, id) || {name:id}).name).join(', ') : 'all'), + h('td', {class:'mono small'}, (e.ips || []).join(', ') || '—'), versionCell(e, latest.blacktip, e.updatePending, '/api/edges/' + e.id + '/update', '/api/edges/' + e.id + '/update/log'), h('td', {class:'small dim'}, (e.orgIds || []).length ? e.orgIds.map(id => (byId(orgs, id) || {name:id}).name).join(', ') : 'all'), h('td', {}, h('div', {class:'actions'}, h('button', {class:'ghost', onclick:() => showCmd(e)}, 'Show command'), h('button', {class:'ghost danger', onclick: act(async () => { if (confirm('Delete this edge and all its ports?')) { await api('DELETE', '/api/edges/' + e.id); refresh(); } })}, 'Delete'))))))) : h('div', {class:'empty'}, 'None yet.'))); }; @@ -1063,8 +1097,14 @@ serverViews.settings = async () => { const updNow = h('button', {class:'btn', style:'display:none', onclick: act(async () => { await api('POST', '/api/admin/update/apply'); updNow.style.display = 'none'; - updOut.textContent = 'Updating. Hammerhead restarts in a few seconds; reload this page after that.'; + showPanel(); })}, 'Update now'); + const updBox = h('div', {style:'margin-top:12px'}); + const showPanel = () => { updOut.textContent = 'Updating Hammerhead. The page reconnects by itself after the restart.'; updBox.replaceChildren(updatePanel(() => api('GET', '/api/admin/update/status'), async s => { if (s.stage === 'done') updOut.textContent = 'Updated: Hammerhead is now running ' + s.version + '.'; })); }; + api('GET', '/api/admin/update/status').then(s => { + if (s.running) showPanel(); + else if ((s.lines || []).length) updBox.append(h('button', {class:'ghost', onclick:() => { updBox.replaceChildren(updatePanel(() => api('GET', '/api/admin/update/status'))); }}, 'Show the last update log')); + }).catch(() => {}); const checkUpdate = act(async () => { updOut.textContent = 'Checking…'; updNow.style.display = 'none'; const u = await api('GET', '/api/admin/update'); @@ -1081,7 +1121,7 @@ serverViews.settings = async () => { h('div', {class:'grid'}, field('SMTP host', f.host), field('Port', f.port), field('Security', f.mode), field('Username', f.user), field('Password', f.pass), field('From', f.from)), h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick: act(async () => { await save(); await api('POST', '/api/admin/settings/test-mail'); toast('Test email sent to ' + S.me.email); })}, 'Save and send a test email'))), card('Updates', h('p', {class:'dim'}, 'Hammerhead, Blacktip and Bamboo can update themselves from new releases when auto-update is on (HAMMERHEAD_AUTO_UPDATE=1). This checks the newest release for Hammerhead; it installs nothing.'), - updOut, h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick:checkUpdate}, 'Check for updates'), updNow)), + updOut, h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick:checkUpdate}, 'Check for updates'), updNow), updBox), h('button', {class:'btn', onclick:save}, 'Save settings')); };