diff --git a/README.md b/README.md index 0fbf7d6..93a4670 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Releases are published for x86-64 only; for ARM machines build it yourself (`TAR Set `HAMMERHEAD_AUTO_UPDATE=1` (or pass `-auto-update`) and Hammerhead checks this server for a newer release every few hours, verifies it against `SHA256SUMS`, checks that the new binary runs, swaps it in and restarts itself. It only follows full releases (never pre-releases), never downgrades, and keeps the old binary if anything looks wrong. `setup.sh` turns it on. It is not available on Windows, and it is for the systemd and plain-binary installs: with Docker, pull a new image instead. -Edges and sites report their version, so Server → Edges and Sites show which release each one runs and flag a newer one. An **Update** button appears for any machine that runs with auto-update on (`BLACKTIP_AUTO_UPDATE=1` / `BAMBOO_AUTO_UPDATE=1`): it asks that machine to install the newest release now, and the result is shown in the table. Under Server → Settings, **Check for updates** does the same lookup for Hammerhead itself, with an **Update now** button when auto-update is on. A machine without auto-update never installs anything on request. +Edges and sites report their version, so Server → Edges and Sites show which release each one runs and flag a newer one. An **Update** button appears for any machine that runs with auto-update on (`BLACKTIP_AUTO_UPDATE=1` / `BAMBOO_AUTO_UPDATE=1`): it asks that machine to install the newest release now, and the result is shown in the table. Under Server → Settings, **Check for updates** does the same lookup for Hammerhead itself, with an **Update now** button when auto-update is on. A machine without auto-update never installs anything on request. Starting an update opens a progress bar with the live log of what the machine is doing (checking, downloading, verifying, installing, restarting). Hammerhead keeps the log of its own last update in `update.log` in its data folder, so it is still there after the restart. **With Docker** diff --git a/agent.go b/agent.go index 23abfe0..b62b621 100644 --- a/agent.go +++ b/agent.go @@ -111,7 +111,9 @@ func (s *Server) edgeReport(w http.ResponseWriter, r *http.Request) { } e.LastSeen = time.Now() e.IPs = in.IPs + oldVersion := e.Version answered := noteUpdateReport(&e.Version, &e.AutoUpdate, &e.UpdateID, &e.UpdateMsg, in.Version, in.AutoUpdate, in.UpdateID, in.UpdateMsg) + s.recordUpdateProgress(e.ID, in.UpdateID, in.UpdateStage, in.UpdatePct, in.UpdateLog, oldVersion, e.Version) for _, x := range s.st.MailEgress { if x.EdgeID == e.ID { if chk, found := in.MailChecks[x.ID]; found { @@ -253,7 +255,9 @@ func (s *Server) siteReport(w http.ResponseWriter, r *http.Request) { in.Connected = []string{} } st.Connected = in.Connected + oldVersion := st.Version answered := noteUpdateReport(&st.Version, &st.AutoUpdate, &st.UpdateID, &st.UpdateMsg, in.Version, in.AutoUpdate, in.UpdateID, in.UpdateMsg) + s.recordUpdateProgress(st.ID, in.UpdateID, in.UpdateStage, in.UpdatePct, in.UpdateLog, oldVersion, st.Version) s.containers[st.ID] = in.Containers for _, x := range s.st.MailEgress { if x.SiteID != st.ID { diff --git a/e2e/updates_test.go b/e2e/updates_test.go index 70ed115..fd34956 100644 --- a/e2e/updates_test.go +++ b/e2e/updates_test.go @@ -58,12 +58,23 @@ func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) { var edgeCalls, siteCalls atomic.Int32 siteCfgHook = func(c *bamboo.Config) { c.Version, c.AutoUpdate = "v0.1.0", true - c.UpdateNow = func(ctx context.Context) (string, error) { siteCalls.Add(1); return "", nil } + c.UpdateNow = func(ctx context.Context, progress func(string, int, string)) (string, error) { + siteCalls.Add(1) + progress("checking", 10, "Checking for the newest release") + progress("done", 100, "Already on the newest release") + return "", nil + } } defer func() { siteCfgHook = nil }() w := setup(t, blacktip.TraefikConfig{}, func(c *blacktip.Config) { c.Version, c.AutoUpdate = "v0.1.0", true - c.UpdateNow = func(ctx context.Context) (string, error) { edgeCalls.Add(1); return "v0.2.0", nil } + c.UpdateNow = func(ctx context.Context, progress func(string, int, string)) (string, error) { + edgeCalls.Add(1) + progress("downloading", 40, "Downloading blacktip_v0.2.0_linux_amd64.tar.gz") + progress("downloading", 70, "Downloaded 28.0 of 40.0 MB (70%)") + progress("restarting", 100, "Installed v0.2.0; restarting into it") + return "v0.2.0", nil + } }) waitFor(t, "the edge to report its version", func() bool { e := w.edgeRow(); return e.Version == "v0.1.0" && e.AutoUpdate }) @@ -77,6 +88,17 @@ func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) { if edgeCalls.Load() != 1 { t.Fatalf("the edge ran its update %d times, want 1", edgeCalls.Load()) } + var log struct { + Running bool + Stage string + Pct int + Lines []string + } + w.admin.must("GET", "/api/edges/"+w.edge+"/update/log", nil, &log) + text := strings.Join(log.Lines, "; ") + if log.Running || log.Stage != "restarting" || log.Pct != 100 || !strings.Contains(text, "Update requested") || !strings.Contains(text, "Downloaded 28.0 of 40.0 MB") || !strings.Contains(text, "restarting into it") { + t.Fatalf("edge update log: %+v (%s)", log, text) + } w.admin.must("POST", "/api/orgs/"+w.org+"/sites/"+w.site+"/update", nil, nil) waitFor(t, "the site to answer the update request", func() bool { @@ -86,6 +108,10 @@ func TestEdgesAndSitesReportVersionsAndUpdateOnRequest(t *testing.T) { if siteCalls.Load() != 1 { t.Fatalf("the site ran its update %d times, want 1", siteCalls.Load()) } + w.admin.must("GET", "/api/orgs/"+w.org+"/sites/"+w.site+"/update/log", nil, &log) + if log.Running || log.Stage != "done" || !strings.Contains(strings.Join(log.Lines, "; "), "Already on the newest release") { + t.Fatalf("site update log: %+v", log) + } // asking again works and is a fresh request, not a leftover answer w.admin.must("POST", "/api/edges/"+w.edge+"/update", nil, nil) diff --git a/go.mod b/go.mod index 9d7f594..9a19468 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.veillax.com/Veillax/hammerhead go 1.27.3 require ( - git.veillax.com/Veillax/bamboo v0.1.4 - git.veillax.com/Veillax/blacktip v0.1.6 - git.veillax.com/Veillax/reef v0.1.6 + git.veillax.com/Veillax/bamboo v0.1.5 + git.veillax.com/Veillax/blacktip v0.1.7 + git.veillax.com/Veillax/reef v0.1.7 github.com/emersion/go-msgauth v0.7.0 golang.org/x/crypto v0.58.0 modernc.org/sqlite v1.61.0 diff --git a/go.sum b/go.sum index 4db7313..c194553 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,16 @@ git.veillax.com/Veillax/bamboo v0.1.3 h1:A66gz9GmmtUq1iIGUAL2ItIeC6iTy5pvqJOoIjR git.veillax.com/Veillax/bamboo v0.1.3/go.mod h1:UuHV6aNHmV9jpow0xeGlXX0QhMB9glPJUGzgIM7S1MM= git.veillax.com/Veillax/bamboo v0.1.4 h1:pgiBJA46CBJx7T3bloCjPekb6wl6W5H8teokFAH5gj0= git.veillax.com/Veillax/bamboo v0.1.4/go.mod h1:wzSpfql7U4a3b9mCVH01OPG9H9szrzwSFDufWw02e0g= +git.veillax.com/Veillax/bamboo v0.1.5 h1:i81c75/7wjayP9tOQ3tSnJWTBhvbjAwNBEPhT+3P2Xk= +git.veillax.com/Veillax/bamboo v0.1.5/go.mod h1:se4kkcP6c3uCGD4HQd+nDLoblS0lN8qa+eFWLo5Horw= git.veillax.com/Veillax/blacktip v0.1.0 h1:L8WpW56gqtvZ1IAapE95Ynl6NKjqTA98WwmEH01kFFk= git.veillax.com/Veillax/blacktip v0.1.0/go.mod h1:sj+mr8393rajzWbl0+RRZrstGyDtZ/GHsut6ebScVxg= git.veillax.com/Veillax/blacktip v0.1.4 h1:o3uNoA0ucB/qcN9ccofysdFA6RTEU4jKdgeTWuE2Uw0= git.veillax.com/Veillax/blacktip v0.1.4/go.mod h1:f7zXCYlQf/9ffDLjPrMHylKAyXX4Nb+aQ7kErmOOhqI= git.veillax.com/Veillax/blacktip v0.1.6 h1:dUJzNGXgieWX05ftWjnMfh6norhaeKTqo1P13lJvjtk= git.veillax.com/Veillax/blacktip v0.1.6/go.mod h1:ZnPRc2gPh9ElILfPH39G/x3B6KAiHQyeoDLycSDgVPM= +git.veillax.com/Veillax/blacktip v0.1.7 h1:PjxEkhE4fnGWht9Pj2/l6S8fOX2Iqjir3rc9vfpMFiQ= +git.veillax.com/Veillax/blacktip v0.1.7/go.mod h1:dVm4wC6wufPjFzK8pvHnqNJJlxroRYmnq8mdkCqOKGc= git.veillax.com/Veillax/reef v0.1.0 h1:O0OhEKFx2CsvACJ20sfzJNjGsngomtfDdKdxu4ROpCs= git.veillax.com/Veillax/reef v0.1.0/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso= git.veillax.com/Veillax/reef v0.1.3 h1:RxSq2xwaup1+MieAkhlvOTZnYwPf19thFHEAe/7VxsI= @@ -18,6 +22,8 @@ git.veillax.com/Veillax/reef v0.1.4 h1:T2+B8P1GAitEyTYRkwMT3UHbG9mN5zXb/inE2muXd git.veillax.com/Veillax/reef v0.1.4/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso= git.veillax.com/Veillax/reef v0.1.6 h1:NgZ4u2Z3hJ+BYD/zX9lov8fox4s2i8txtYNOpIsikVo= git.veillax.com/Veillax/reef v0.1.6/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso= +git.veillax.com/Veillax/reef v0.1.7 h1:5C2udonlJac2xvDOE5e+zDxej+AbZ9ttrNSE4owmwuU= +git.veillax.com/Veillax/reef v0.1.7/go.mod h1:ZBUd/BARO5kkpUNTP81TTY1aQXLHFR6bKDhCxckjHso= github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g= github.com/Microsoft/go-winio v0.6.4/go.mod h1:1xl5JXG4ze/m74GH6Dc4Mwxfc3VL5siRVQjIAlHH07U= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= diff --git a/server.go b/server.go index 00866e6..661c99f 100644 --- a/server.go +++ b/server.go @@ -99,7 +99,10 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /api/admin/settings/test-mail", s.withServerAdmin(s.testMail)) mux.HandleFunc("GET /api/admin/update", s.withServerAdmin(s.checkUpdate)) mux.HandleFunc("POST /api/admin/update/apply", s.withServerAdmin(s.applyUpdate)) + mux.HandleFunc("GET /api/admin/update/status", s.withServerAdmin(s.updateStatus)) mux.HandleFunc("GET /api/versions", s.withUser(s.latestVersions)) + mux.HandleFunc("GET /api/edges/{id}/update/log", s.withServerAdmin(s.edgeUpdateLog)) + mux.HandleFunc("GET /api/orgs/{org}/sites/{id}/update/log", s.withOrg(true, s.siteUpdateLog)) mux.HandleFunc("POST /api/edges/{id}/update", s.withServerAdmin(s.updateEdge)) mux.HandleFunc("POST /api/orgs/{org}/sites/{id}/update", s.withOrg(true, s.updateSite)) mux.HandleFunc("GET /api/admin/audit", s.withServerAdmin(func(w http.ResponseWriter, r *http.Request, p *principal) { diff --git a/store.go b/store.go index 62de4c9..6a08c15 100644 --- a/store.go +++ b/store.go @@ -209,6 +209,8 @@ type Server struct { oidc map[string]*oidcDoc alerts map[string]*alertState // active problems by key (see alerts.go) + updRuns map[string]*updRun // progress of updates running on edges and sites, by ID + selfUpd selfUpdate // progress of this server's own update verMu sync.Mutex // guards verCache and verAt, the latest releases seen by /api/versions verCache map[string]string verAt time.Time @@ -253,6 +255,7 @@ func New(dir string, opts Options) (*Server, error) { oidc: map[string]*oidcDoc{}, alerts: map[string]*alertState{}, mtaSent: map[string]time.Time{}, + updRuns: map[string]*updRun{}, } s.st.Sessions = map[string]*Session{} s.st.Secrets = map[string]*Secret{} @@ -282,6 +285,7 @@ func New(dir string, opts Options) (*Server, error) { if opts.SelfDomain != "" && opts.SelfTunnel == "" { s.opts.SelfTunnel = opts.SelfDomain + ":51820" } + s.finishSelfUpdateLog() if err := s.ensureSelfEdge(); err != nil { db.Close() return nil, err diff --git a/update.go b/update.go index d9e1743..393e95f 100644 --- a/update.go +++ b/update.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "os" "strings" "sync" "time" @@ -45,13 +46,32 @@ func (s *Server) applyUpdate(w http.ResponseWriter, r *http.Request, p *principa fail(w, 400, errors.New("auto-update is off: set HAMMERHEAD_AUTO_UPDATE=1 and restart Hammerhead, or update it by hand")) return } + u := &s.selfUpd + u.mu.Lock() + if u.running { + u.mu.Unlock() + fail(w, 409, errors.New("an update is already running")) + return + } + u.running, u.stage, u.pct, u.lines = true, "starting", 0, nil + u.mu.Unlock() + os.WriteFile(s.updateLogPath(), nil, 0o600) // a new run starts a new log cfg := s.updateConfig("hammerhead", s.opts.Version) - cfg.Restart = func(exe string) error { s.Close(); return selfupdate.Restart(exe) } + cfg.Progress = func(e selfupdate.Event) { s.selfNote(e.Stage, e.Pct, e.Line) } + cfg.Restart = func(exe string) error { + s.selfNote("restarting", 100, "Closing the database and restarting") + s.Close() + return selfupdate.Restart(exe) + } + s.selfNote("starting", 0, "Update requested by "+p.User.Email) s.audit("", p.User.Email, "server.update", "") go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() selfupdate.CheckOnce(ctx, cfg) // on success the process is replaced and this never returns + u.mu.Lock() + u.running = false + u.mu.Unlock() }() writeJSON(w, 200, map[string]any{"started": true}) } @@ -121,6 +141,7 @@ func (s *Server) updateEdge(w http.ResponseWriter, r *http.Request, p *principal fail(w, 400, err) return } + delete(s.updRuns, e.ID) // a new request starts a new log s.bump() s.audit("", p.User.Email, "edge.update", e.Name) ok(w) @@ -142,6 +163,7 @@ func (s *Server) updateSite(w http.ResponseWriter, r *http.Request, c *orgCtx) { fail(w, 400, err) return } + delete(s.updRuns, st.ID) s.bump() s.audit(c.Org.ID, c.Actor, "site.update", st.Name) ok(w) diff --git a/update_test.go b/update_test.go index 1057df9..aa63d43 100644 --- a/update_test.go +++ b/update_test.go @@ -5,6 +5,9 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" ) @@ -52,3 +55,36 @@ func TestCheckForUpdates(t *testing.T) { t.Errorf("unreachable server: %d %s", code, b) } } + +func TestSelfUpdateLogSurvivesTheRestartItEndsWith(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "update.log"), []byte("12:00:00 Checking\n12:00:09 Installed v0.3.0; restarting into it\n"), 0o600); err != nil { + t.Fatal(err) + } + s, err := New(dir, Options{AdminEmail: "admin@test.local", AdminPassword: "password123", MasterKey: key, Version: "v0.3.0"}) + if err != nil { + t.Fatal(err) + } + defer s.Close() + do := apiClient(t, s) + code, b := do("GET", "/api/admin/update/status", nil) + var out struct { + Running bool + Stage string + Pct int + Lines []string + Version string + } + json.Unmarshal(b, &out) + last := "" + if len(out.Lines) > 0 { + last = out.Lines[len(out.Lines)-1] + } + if code != 200 || out.Running || out.Stage != "done" || out.Pct != 100 || len(out.Lines) != 3 || !strings.Contains(last, "now running v0.3.0") { + t.Fatalf("status: %d %s", code, b) + } + // starting an update needs auto-update on + if code, b := do("POST", "/api/admin/update/apply", nil); code != 400 || !strings.Contains(string(b), "HAMMERHEAD_AUTO_UPDATE") { + t.Fatalf("apply without auto-update: %d %s", code, b) + } +} diff --git a/updatelog.go b/updatelog.go new file mode 100644 index 0000000..440eddc --- /dev/null +++ b/updatelog.go @@ -0,0 +1,152 @@ +package hammerhead + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const maxUpdateLines = 200 + +// updRun is the progress of one update an edge or site is carrying out, as that agent reported it. +type updRun struct { + ID string // the update request it belongs to + Stage string + Pct int + Lines []string +} + +// recordUpdateProgress keeps what an agent says about the update it is running. When the agent +// reports a new version, the update is finished: its log gets a closing line and is then left alone, +// so the freshly restarted program (which starts a new, empty log) can't overwrite it. +// Caller holds s.mu. +func (s *Server) recordUpdateProgress(agentID, reqID, stage string, pct int, lines []string, oldVersion, newVersion string) { + run := s.updRuns[agentID] + if run != nil && run.Stage != "done" && oldVersion != "" && newVersion != "" && newVersion != oldVersion { + run.Lines = append(run.Lines, time.Now().Format("15:04:05")+" Now running "+newVersion) + run.Stage, run.Pct = "done", 100 + return + } + if reqID == "" || (stage == "" && len(lines) == 0) { + return + } + if run != nil && run.ID == reqID && run.Stage == "done" { + return + } + s.updRuns[agentID] = &updRun{ID: reqID, Stage: stage, Pct: pct, Lines: lines} +} + +func (s *Server) updateLogView(agentID string, pending bool, msg, version string) map[string]any { + out := map[string]any{"running": pending, "msg": msg, "version": version, "stage": "", "pct": 0, "lines": []string{}} + if run := s.updRuns[agentID]; run != nil { + out["stage"], out["pct"], out["lines"] = run.Stage, run.Pct, run.Lines + } + return out +} + +func (s *Server) edgeUpdateLog(w http.ResponseWriter, r *http.Request, p *principal) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.findEdge(r.PathValue("id")) + if e == nil { + fail(w, 404, errors.New("no such edge")) + return + } + writeJSON(w, 200, s.updateLogView(e.ID, e.UpdateID != "", e.UpdateMsg, e.Version)) +} + +func (s *Server) siteUpdateLog(w http.ResponseWriter, r *http.Request, c *orgCtx) { + s.mu.Lock() + defer s.mu.Unlock() + st := s.findSite(r.PathValue("id")) + if st == nil || !s.siteVisible(c, st) { + fail(w, 404, errors.New("no such site")) + return + } + writeJSON(w, 200, s.updateLogView(st.ID, st.UpdateID != "", st.UpdateMsg, st.Version)) +} + +// selfUpdate is the progress of this server's own update. It is kept in memory and in /update.log, +// so the page can still show it after the restart that ends a successful update. +type selfUpdate struct { + mu sync.Mutex + running bool + stage string + pct int + lines []string +} + +func (s *Server) updateLogPath() string { return filepath.Join(s.dir, "update.log") } + +func (s *Server) selfNote(stage string, pct int, line string) { + u := &s.selfUpd + u.mu.Lock() + defer u.mu.Unlock() + if stage != "" { + u.stage = stage + } + if pct >= 0 { + u.pct = pct + } + if line == "" { + return + } + line = time.Now().Format("15:04:05") + " " + line + u.lines = append(u.lines, line) + if len(u.lines) > maxUpdateLines { + u.lines = u.lines[len(u.lines)-maxUpdateLines:] + } + if f, err := os.OpenFile(s.updateLogPath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600); err == nil { + f.WriteString(line + "\n") + f.Close() + } +} + +func readLines(path string) []string { + b, err := os.ReadFile(path) + if err != nil { + return nil + } + lines := strings.Split(strings.TrimRight(string(b), "\n"), "\n") + if len(lines) == 1 && lines[0] == "" { + return nil + } + if len(lines) > maxUpdateLines { + lines = lines[len(lines)-maxUpdateLines:] + } + return lines +} + +// finishSelfUpdateLog runs at startup: if the last run ended by restarting into this process, say so. +func (s *Server) finishSelfUpdateLog() { + lines := readLines(s.updateLogPath()) + if n := len(lines); n > 0 && strings.Contains(lines[n-1], "restarting") && s.opts.Version != "" { + if f, err := os.OpenFile(s.updateLogPath(), os.O_APPEND|os.O_WRONLY, 0o600); err == nil { + f.WriteString(time.Now().Format("15:04:05") + " Hammerhead restarted: now running " + s.opts.Version + "\n") + f.Close() + } + } +} + +// updateStatus returns the progress and log of this server's own update, for the Settings page. +func (s *Server) updateStatus(w http.ResponseWriter, r *http.Request, p *principal) { + u := &s.selfUpd + u.mu.Lock() + out := map[string]any{"running": u.running, "stage": u.stage, "pct": u.pct, "lines": append([]string(nil), u.lines...), "version": s.opts.Version} + u.mu.Unlock() + if lines, _ := out["lines"].([]string); len(lines) == 0 { + // nothing in memory: this is a fresh process, so show the last update's log from disk + if saved := readLines(s.updateLogPath()); len(saved) > 0 { + stage := "done" + if strings.Contains(saved[len(saved)-1], "Failed:") { + stage = "failed" + } + out["lines"], out["stage"], out["pct"] = saved, stage, 100 + } + } + writeJSON(w, 200, out) +} diff --git a/web/index.html b/web/index.html index 5f3607c..5080ea8 100644 --- a/web/index.html +++ b/web/index.html @@ -76,6 +76,10 @@ .choice.on { border-color:var(--accent); background:var(--accent-soft); } .choice-t { font-weight:600; } .sect-h { padding:var(--s3) var(--s4); font-weight:600; border-bottom:1px solid var(--border); } + .progress { height:8px; background:var(--surface-2); border-radius:999px; overflow:hidden; } + .progress > div { height:100%; background:var(--accent); transition:width .4s ease; } + .progress > div.bad { background:var(--danger); } + .logbox { margin:0; background:var(--bg); border:1px solid var(--border); border-radius:var(--radius); padding:var(--s3); font:var(--text-xs)/1.6 var(--font-mono); max-height:260px; overflow:auto; white-space:pre-wrap; word-break:break-word; } /* forms */ label { display:flex; flex-direction:column; gap:var(--s1); font-size:var(--text-sm); font-weight:600; color:var(--text); } @@ -191,6 +195,34 @@ const byId = (list, id) => (list || []).find(x => x.id === id); const field = (label, el, hint) => h('label', {}, label, el, hint ? h('span', {class:'sub'}, hint) : null); const chk = (label, checked, onchange) => { const i = h('input', {type:'checkbox', checked: !!checked}); if (onchange) i.onchange = () => onchange(i.checked); return Object.assign(h('label', {class:'chk'}, i, label), {box:i}); }; // ----- versions and updates ----- +// A progress bar and live log for an update. load() returns {running, stage, pct, lines}; the panel polls it +// until the update is over, and keeps going through the gap while a program restarts into its new version. +function updatePanel(load, onDone) { + const bar = h('div', {style:'width:0%'}), stageEl = h('div', {class:'small'}, 'Starting…'), logEl = h('pre', {class:'logbox'}, 'Waiting for the first message…'); + const el = h('div', {class:'stack'}, stageEl, h('div', {class:'progress'}, bar), logEl); + const label = {starting:'Starting', checking:'Checking for the release', downloading:'Downloading', verifying:'Verifying', installing:'Installing', restarting:'Restarting', done:'Done', failed:'Failed'}; + let misses = 0; + const tick = async () => { + if (!el.isConnected) return; // the dialog or card was closed + let again = true; + try { + const s = await load(); misses = 0; + const pct = Math.max(0, Math.min(100, s.pct || 0)), lines = s.lines || []; + bar.style.width = pct + '%'; bar.className = s.stage === 'failed' ? 'bad' : ''; + stageEl.textContent = (label[s.stage] || s.stage || 'Waiting') + (pct ? ' · ' + pct + '%' : ''); + if (lines.length) { const atEnd = logEl.scrollTop + logEl.clientHeight >= logEl.scrollHeight - 4; logEl.textContent = lines.join('\n'); if (atEnd) logEl.scrollTop = logEl.scrollHeight; } + if (!s.running && (s.stage === 'done' || s.stage === 'failed')) { again = false; if (onDone) onDone(s); } + } catch (e) { + misses++; // the server is probably restarting into the new version + if (misses > 2) stageEl.textContent = 'Restarting… waiting for it to come back'; + if (misses > 150) { again = false; stageEl.textContent = 'It has not come back after a few minutes. Check its log on the server.'; } + } + if (again) setTimeout(tick, 800); + }; + setTimeout(tick, 100); + return el; +} + // compares vX.Y.Z strings: -1 if a is older than b, 1 if newer, 0 if equal or not comparable const cmpVer = (a, b) => { const p = v => (/^v(\d+)\.(\d+)\.(\d+)$/.exec(v || '') || []).slice(1).map(Number), x = p(a), y = p(b); @@ -201,7 +233,8 @@ const cmpVer = (a, b) => { const latestVersions = async () => { try { return await api('GET', '/api/versions'); } catch (e) { return {}; } }; // A table cell with a machine's version, a note when a newer release exists, and an Update button // when that machine allows being updated from here (it must run with auto-update on). -const versionCell = (m, latest, pending, updateUrl) => { +const updateDialog = (m, logUrl) => modal('Updating ' + m.name, h('div', {class:'stack'}, updatePanel(() => api('GET', logUrl), () => refresh()), h('p', {class:'small dim'}, 'You can close this; the update carries on.'))); +const versionCell = (m, latest, pending, updateUrl, logUrl) => { const behind = latest && m.version && cmpVer(m.version, latest) < 0, kids = [h('span', {class:'mono small'}, m.version || 'unknown')]; if (behind) kids.push(h('div', {class:'small'}, latest + ' is available')); if (pending) { @@ -210,7 +243,8 @@ const versionCell = (m, latest, pending, updateUrl) => { } else if (m.updateMsg) kids.push(h('div', {class:'small dim'}, m.updateMsg)); if (behind && !pending) kids.push(h('button', {class:'ghost', disabled:!m.autoUpdate, title:m.autoUpdate ? 'Install ' + latest + ' now' : 'Turn on auto-update on that machine (…_AUTO_UPDATE=1) to update it from here', - onclick: act(async () => { await api('POST', updateUrl); toast('Update requested'); refresh(); })}, 'Update')); + onclick: act(async () => { await api('POST', updateUrl); updateDialog(m, logUrl); refresh(); })}, 'Update')); + if (pending || m.updateMsg) kids.push(h('button', {class:'ghost', onclick:() => updateDialog(m, logUrl)}, 'Log')); return h('td', {}, h('div', {class:'stack'}, ...kids)); }; @@ -719,7 +753,7 @@ views.sites = async () => { h('td', {}, h('span', {class:'dot ' + (st.online[s.id] ? 'ok' : 'bad')})), h('td', {}, s.name), h('td', {}, s.local ? 'this machine (direct)' : 'bamboo agent'), h('td', {}, s.local ? '—' : ((s.connected || []).map(id => (byId(st.edges, id) || {name:id}).name).join(', ') || 'nothing connected')), - s.local ? h('td', {class:'dim'}, '—') : versionCell(s, latest.bamboo, !!s.updateId, O(`/sites/${s.id}/update`)), + s.local ? h('td', {class:'dim'}, '—') : versionCell(s, latest.bamboo, !!s.updateId, O(`/sites/${s.id}/update`), O(`/sites/${s.id}/update/log`)), h('td', {}, h('button', {class:'ghost', onclick: act(async () => { const l = await api('GET', O(`/sites/${s.id}/containers`)); modal('Containers on ' + s.name, l.length ? h('table', {}, h('thead', {}, h('tr', {}, ['Name', 'Image', 'Ports'].map(t => h('th', {}, t)))), h('tbody', {}, l.map(c => h('tr', {}, h('td', {}, c.name), h('td', {class:'dim'}, c.image), h('td', {class:'mono small'}, c.ports.map(p => (p.public ? p.public + '→' : '') + p.private + '/' + p.type).join(', ')))))) : @@ -994,7 +1028,7 @@ serverViews.edges = async () => { return h('div', {}, card('Edges', edges.length ? h('table', {}, h('thead', {}, h('tr', {}, ['', 'Name', 'WireGuard endpoint', 'IP addresses', 'Version', 'Orgs', ''].map(t => h('th', {}, t)))), h('tbody', {}, edges.map(e => h('tr', {}, h('td', {}, h('span', {class:'dot ' + (e.online ? 'ok' : 'bad'), title:e.online ? 'online' : 'offline'})), h('td', {}, e.name), h('td', {class:'mono'}, e.tunnelAddr), - h('td', {class:'mono small'}, (e.ips || []).join(', ') || '—'), versionCell(e, latest.blacktip, e.updatePending, '/api/edges/' + e.id + '/update'), h('td', {class:'small dim'}, (e.orgIds || []).length ? e.orgIds.map(id => (byId(orgs, id) || {name:id}).name).join(', ') : 'all'), + h('td', {class:'mono small'}, (e.ips || []).join(', ') || '—'), versionCell(e, latest.blacktip, e.updatePending, '/api/edges/' + e.id + '/update', '/api/edges/' + e.id + '/update/log'), h('td', {class:'small dim'}, (e.orgIds || []).length ? e.orgIds.map(id => (byId(orgs, id) || {name:id}).name).join(', ') : 'all'), h('td', {}, h('div', {class:'actions'}, h('button', {class:'ghost', onclick:() => showCmd(e)}, 'Show command'), h('button', {class:'ghost danger', onclick: act(async () => { if (confirm('Delete this edge and all its ports?')) { await api('DELETE', '/api/edges/' + e.id); refresh(); } })}, 'Delete'))))))) : h('div', {class:'empty'}, 'None yet.'))); }; @@ -1063,8 +1097,14 @@ serverViews.settings = async () => { const updNow = h('button', {class:'btn', style:'display:none', onclick: act(async () => { await api('POST', '/api/admin/update/apply'); updNow.style.display = 'none'; - updOut.textContent = 'Updating. Hammerhead restarts in a few seconds; reload this page after that.'; + showPanel(); })}, 'Update now'); + const updBox = h('div', {style:'margin-top:12px'}); + const showPanel = () => { updOut.textContent = 'Updating Hammerhead. The page reconnects by itself after the restart.'; updBox.replaceChildren(updatePanel(() => api('GET', '/api/admin/update/status'), async s => { if (s.stage === 'done') updOut.textContent = 'Updated: Hammerhead is now running ' + s.version + '.'; })); }; + api('GET', '/api/admin/update/status').then(s => { + if (s.running) showPanel(); + else if ((s.lines || []).length) updBox.append(h('button', {class:'ghost', onclick:() => { updBox.replaceChildren(updatePanel(() => api('GET', '/api/admin/update/status'))); }}, 'Show the last update log')); + }).catch(() => {}); const checkUpdate = act(async () => { updOut.textContent = 'Checking…'; updNow.style.display = 'none'; const u = await api('GET', '/api/admin/update'); @@ -1081,7 +1121,7 @@ serverViews.settings = async () => { h('div', {class:'grid'}, field('SMTP host', f.host), field('Port', f.port), field('Security', f.mode), field('Username', f.user), field('Password', f.pass), field('From', f.from)), h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick: act(async () => { await save(); await api('POST', '/api/admin/settings/test-mail'); toast('Test email sent to ' + S.me.email); })}, 'Save and send a test email'))), card('Updates', h('p', {class:'dim'}, 'Hammerhead, Blacktip and Bamboo can update themselves from new releases when auto-update is on (HAMMERHEAD_AUTO_UPDATE=1). This checks the newest release for Hammerhead; it installs nothing.'), - updOut, h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick:checkUpdate}, 'Check for updates'), updNow)), + updOut, h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick:checkUpdate}, 'Check for updates'), updNow), updBox), h('button', {class:'btn', onclick:save}, 'Save settings')); };