Add self-update, including in Docker

Sandbar can install new releases itself using Reef's selfupdate, automatically with
SANDBAR_AUTO_UPDATE=1 or from a button on the Account page. In a container the new binary is
written to <data>/bin and takes over from the image at start, until the image catches up.
Tags now publish a Gitea release with archives and SHA256SUMS as well as the image.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/sandbar/api.go

@@ -105,6 +105,8 @@ func (s *Server) Handler() http.Handler {
 		}
 		return nil
 	}, nil)
+	mux.HandleFunc("GET /api/v1/system/update", s.auth(true, s.updateStatus))
+	mux.HandleFunc("POST /api/v1/system/update", s.auth(true, s.updateApply))
 	mux.HandleFunc("POST /api/v1/channels/{id}/test", s.auth(true, s.testChannel))
 	mux.HandleFunc("GET /api/v1/oncall", s.auth(false, s.onCallNow))
 

internal/sandbar/server.go

@@ -26,6 +26,15 @@ type Options struct {
 	MessageDays  int // how long topic messages are kept (default 30)
 	CheckDays    int // how long monitor results are kept (default 30)
 
+	// Self-update (see update.go). Version is the running release, such as v0.2.0; anything else
+	// ("dev") never updates. Restart is called after a new binary is installed and should close
+	// what the process holds open before re-executing it.
+	Version      string
+	AutoUpdate   bool
+	UpdateServer string // default https://git.veillax.com
+	UpdateRepo   string // default Veillax/sandbar
+	Restart      func(exe string) error
+
 	// Tests replace these.
 	Now        func() time.Time
 	HTTPClient *http.Client
@@ -44,6 +53,8 @@ type Server struct {
 	loginMu  sync.Mutex
 	loginTry map[string][]time.Time
 
+	upd updateState
+
 	stop chan struct{}
 	wg   sync.WaitGroup
 }

internal/sandbar/update.go

@@ -0,0 +1,255 @@
+package sandbar
+
+import (
+	"context"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"git.veillax.com/Veillax/reef/selfupdate"
+)
+
+// Self-update replaces the running binary with the newest release published on the Gitea
+// server (Reef's selfupdate does the download, checksum and swap).
+//
+// In a container the binary lives in the image, where this user can't write and where a
+// change would be lost when the container is recreated. So an update is written to
+// <data>/bin/sandbar on the data volume instead, and every start checks that file first
+// (UseNewerOverlay): it runs when it is a newer release than the image, and is deleted once
+// the image has caught up.
+
+const defaultUpdateServer = "https://git.veillax.com"
+
+type updateState struct {
+	mu      sync.Mutex
+	running bool
+	stage   string
+	pct     int
+	line    string
+	err     string
+
+	checked  time.Time // the cached answer from the release server
+	latest   string
+	newer    bool
+	checkErr string
+}
+
+func (s *Server) updateConfig() selfupdate.Config {
+	o := s.opts
+	server, repo := o.UpdateServer, o.UpdateRepo
+	if server == "" {
+		server = defaultUpdateServer
+	}
+	if repo == "" {
+		repo = "Veillax/sandbar"
+	}
+	return selfupdate.Config{
+		Server: server, Repo: repo, App: "sandbar", Current: o.Version,
+		ExePath: UpdateTarget(o.DataDir), Restart: o.Restart, Logf: log.Printf,
+	}
+}
+
+// RunAutoUpdate checks for a new release shortly after start and then every few hours until
+// ctx ends, installing what it finds and restarting into it.
+func (s *Server) RunAutoUpdate(ctx context.Context) {
+	selfupdate.Run(ctx, s.updateConfig())
+}
+
+// canInstall says whether this build can update itself, and if not, why.
+func (s *Server) canInstall() (bool, string) {
+	if _, ok := parseVersion(s.opts.Version); !ok {
+		return false, "This is a development build, so it does not update itself."
+	}
+	if runtime.GOOS == "windows" {
+		return false, "A running program can't be replaced on Windows. Download the new release and restart."
+	}
+	return true, ""
+}
+
+// updateStatus reports the running and newest versions. The release server is asked at most
+// every ten minutes unless ?fresh=1.
+func (s *Server) updateStatus(w http.ResponseWriter, r *http.Request) {
+	ok, why := s.canInstall()
+	u := &s.upd
+	u.mu.Lock()
+	stale := time.Since(u.checked) > 10*time.Minute || r.URL.Query().Get("fresh") == "1"
+	running := u.running
+	u.mu.Unlock()
+	if stale && !running {
+		ctx, cancel := context.WithTimeout(r.Context(), 8*time.Second)
+		latest, newer, err := selfupdate.Check(ctx, s.updateConfig())
+		cancel()
+		u.mu.Lock()
+		u.checked, u.latest, u.newer, u.checkErr = time.Now(), latest, newer, ""
+		if err != nil {
+			u.checkErr = err.Error()
+		}
+		u.mu.Unlock()
+	}
+	u.mu.Lock()
+	out := map[string]any{
+		"current": s.opts.Version, "autoUpdate": s.opts.AutoUpdate, "canInstall": ok, "reason": why,
+		"latest": u.latest, "newer": u.newer, "error": u.checkErr,
+		"running": u.running, "stage": u.stage, "pct": u.pct, "line": u.line, "failed": u.err,
+	}
+	u.mu.Unlock()
+	writeJSON(w, 200, out)
+}
+
+// updateApply installs the newest release now. It answers at once; the page polls updateStatus
+// for progress, and the server disappears for a moment when it restarts into the new binary.
+func (s *Server) updateApply(w http.ResponseWriter, r *http.Request) {
+	if ok, why := s.canInstall(); !ok {
+		bad(w, why)
+		return
+	}
+	u := &s.upd
+	u.mu.Lock()
+	if u.running {
+		u.mu.Unlock()
+		writeJSON(w, 409, map[string]string{"error": "An update is already running."})
+		return
+	}
+	u.running, u.stage, u.pct, u.line, u.err = true, "starting", 0, "", ""
+	u.mu.Unlock()
+	cfg := s.updateConfig()
+	cfg.Progress = func(e selfupdate.Event) {
+		u.mu.Lock()
+		defer u.mu.Unlock()
+		u.stage = e.Stage
+		if e.Pct >= 0 {
+			u.pct = e.Pct
+		}
+		if e.Line != "" {
+			u.line = e.Line
+		}
+	}
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+		defer cancel()
+		_, err := selfupdate.CheckOnce(ctx, cfg) // when it installs, the process is replaced and this never returns
+		u.mu.Lock()
+		u.running = false
+		if err != nil {
+			u.err = err.Error()
+		}
+		u.checked = time.Time{} // look again next time
+		u.mu.Unlock()
+	}()
+	writeJSON(w, 200, map[string]bool{"started": true})
+}
+
+// UpdateTarget is the file an update replaces: the running binary when its folder is writable
+// (a plain install), and otherwise <data>/bin/sandbar (a container).
+func UpdateTarget(dataDir string) string {
+	if exe, err := os.Executable(); err == nil {
+		if exe, err = filepath.EvalSymlinks(exe); err == nil && writableDir(filepath.Dir(exe)) {
+			return exe
+		}
+	}
+	dir, err := filepath.Abs(filepath.Join(dataDir, "bin"))
+	if err != nil {
+		dir = filepath.Join(dataDir, "bin")
+	}
+	os.MkdirAll(dir, 0o755)
+	return filepath.Join(dir, "sandbar")
+}
+
+func writableDir(dir string) bool {
+	f, err := os.CreateTemp(dir, ".sandbar-write-test-*")
+	if err != nil {
+		return false
+	}
+	f.Close()
+	os.Remove(f.Name())
+	return true
+}
+
+// UseNewerOverlay runs <data>/bin/sandbar instead of this binary when that file is a newer
+// release. It returns when there is nothing to switch to, and otherwise replaces the process.
+func UseNewerOverlay(dataDir, current string, logf func(string, ...any)) {
+	if runtime.GOOS == "windows" || os.Getenv("SANDBAR_NO_OVERLAY") == "1" {
+		return
+	}
+	overlay, err := filepath.Abs(filepath.Join(dataDir, "bin", "sandbar"))
+	if err != nil {
+		return
+	}
+	ofi, err := os.Stat(overlay)
+	if err != nil {
+		return
+	}
+	if exe, err := os.Executable(); err == nil {
+		if efi, err := os.Stat(exe); err == nil && os.SameFile(efi, ofi) {
+			return // this already is the overlay
+		}
+	}
+	ver := overlayVersion(overlay)
+	switch {
+	case ver == "":
+		logf("ignoring %s: it did not report a version", overlay)
+	case versionNewer(ver, current):
+		logf("running %s from %s, which is newer than this image's %s", ver, overlay, current)
+		if err := selfupdate.Restart(overlay); err != nil {
+			logf("could not switch to %s: %v", overlay, err)
+		}
+	default:
+		if _, ok := parseVersion(current); ok {
+			os.Remove(overlay) // the image has caught up with it
+		}
+	}
+}
+
+// overlayVersion asks a binary for its version, or returns "" if it will not run.
+func overlayVersion(path string) string {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, path, "-version").Output()
+	f := strings.Fields(string(out))
+	if err != nil || len(f) != 2 || f[0] != "sandbar" {
+		return ""
+	}
+	return f[1]
+}
+
+// parseVersion reads vX.Y.Z. Pre-releases and local builds are not versions for this purpose.
+func parseVersion(s string) ([3]int, bool) {
+	var v [3]int
+	if !strings.HasPrefix(s, "v") || strings.ContainsAny(s, "-+") {
+		return v, false
+	}
+	parts := strings.Split(s[1:], ".")
+	if len(parts) != 3 {
+		return v, false
+	}
+	for i, p := range parts {
+		n, err := strconv.Atoi(p)
+		if err != nil || n < 0 {
+			return v, false
+		}
+		v[i] = n
+	}
+	return v, true
+}
+
+// versionNewer reports whether a is a later release than b. Either not being a release is false.
+func versionNewer(a, b string) bool {
+	x, ok1 := parseVersion(a)
+	y, ok2 := parseVersion(b)
+	if !ok1 || !ok2 {
+		return false
+	}
+	for i := range x {
+		if x[i] != y[i] {
+			return x[i] > y[i]
+		}
+	}
+	return false
+}

internal/sandbar/update_test.go

@@ -0,0 +1,109 @@
+package sandbar
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func TestVersionNewer(t *testing.T) {
+	for _, c := range []struct {
+		a, b string
+		want bool
+	}{
+		{"v0.1.4", "v0.1.3", true},
+		{"v0.2.0", "v0.1.9", true},
+		{"v1.0.0", "v0.9.9", true},
+		{"v0.1.3", "v0.1.3", false},
+		{"v0.1.2", "v0.1.3", false},
+		{"v0.1.10", "v0.1.9", true},
+		{"v0.2.0-rc1", "v0.1.0", false}, // pre-releases are never offered
+		{"v0.2.0", "dev", false},        // a development build never updates
+		{"dev", "v0.1.0", false},
+	} {
+		if got := versionNewer(c.a, c.b); got != c.want {
+			t.Errorf("versionNewer(%q, %q) = %v, want %v", c.a, c.b, got, c.want)
+		}
+	}
+}
+
+// The update page asks the release server and reports whether it has something newer.
+func TestUpdateStatus(t *testing.T) {
+	release := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v1/repos/Veillax/sandbar/releases/latest" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Write([]byte(`{"tag_name":"v0.2.0","assets":[]}`))
+	}))
+	defer release.Close()
+
+	s, _, _ := newTestServer(t)
+	s.opts.Version, s.opts.UpdateServer = "v0.1.0", release.URL
+	ts := httptest.NewServer(s.Handler())
+	defer ts.Close()
+	c := newAPIClient(t, ts)
+
+	if code := c.do("GET", "/api/v1/system/update", nil, nil); code != 401 {
+		t.Fatalf("signed out: %d", code)
+	}
+	c.setup()
+	var st map[string]any
+	if code := c.do("GET", "/api/v1/system/update", nil, &st); code != 200 {
+		t.Fatalf("status: %d", code)
+	}
+	if st["current"] != "v0.1.0" || st["latest"] != "v0.2.0" || st["newer"] != true {
+		t.Fatalf("status: %v", st)
+	}
+	if st["error"] != "" {
+		t.Fatalf("unexpected error: %v", st["error"])
+	}
+
+	// A development build can look but not install.
+	s.opts.Version = "dev"
+	s.upd.checked = s.upd.checked.AddDate(-1, 0, 0)
+	if code := c.do("POST", "/api/v1/system/update", map[string]string{}, &st); code != 400 {
+		t.Fatalf("a development build must not install: %d %v", code, st)
+	}
+}
+
+// A newer binary on the data volume takes over from the image, and a stale one is removed.
+func TestOverlay(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("self-update is not supported on Windows")
+	}
+	dir := t.TempDir()
+	bin := filepath.Join(dir, "bin")
+	os.MkdirAll(bin, 0o755)
+	overlay := filepath.Join(bin, "sandbar")
+	script := func(version string) {
+		os.WriteFile(overlay, []byte("#!/bin/sh\necho 'sandbar "+version+"'\n"), 0o755)
+	}
+	var logged []string
+	logf := func(f string, a ...any) { logged = append(logged, f) }
+
+	// An older overlay is deleted once the image is a release that has caught up.
+	script("v0.1.0")
+	UseNewerOverlay(dir, "v0.1.5", logf)
+	if _, err := os.Stat(overlay); err == nil {
+		t.Fatal("a stale overlay should be removed")
+	}
+
+	// A development image never deletes or switches to anything.
+	script("v0.9.0")
+	UseNewerOverlay(dir, "dev", logf)
+	if _, err := os.Stat(overlay); err != nil {
+		t.Fatal("a development build must leave the overlay alone")
+	}
+
+	// Something that does not report a version is ignored, not run.
+	os.WriteFile(overlay, []byte("#!/bin/sh\necho hello\n"), 0o755)
+	UseNewerOverlay(dir, "v0.1.5", logf)
+	if len(logged) == 0 || !strings.Contains(logged[len(logged)-1], "did not report a version") {
+		t.Fatalf("log: %v", logged)
+	}
+}

internal/sandbar/web/index.html

@@ -755,8 +755,36 @@ function passwordDialog(u) {
 }
 
 // ---------- account ----------
+// Administrators see which release is running and can install a newer one. The server restarts
+// into the new binary, so the page waits for it to come back.
+function updateCard(u) {
+  const status = h('p', {style:'margin:0'}), detail = h('p', {class:'dim', style:'margin:0'}), actions = h('div', {class:'actions form-actions'});
+  const paint = u => {
+    status.replaceChildren(`Running ${u.current}. `, u.error ? 'The newest release could not be checked.' : u.newer ? `${u.latest} is available.` : u.latest ? 'This is the newest release.' : '');
+    detail.textContent = u.error ? u.error : !u.canInstall ? u.reason : u.autoUpdate ? 'Automatic updates are on. Sandbar checks every few hours.' : 'Automatic updates are off. Set SANDBAR_AUTO_UPDATE=1 and restart to turn them on.';
+    actions.replaceChildren(...[h('button', {class:'ghost', onclick:act(async () => { paint(await api('GET', V1 + '/system/update?fresh=1')); })}, 'Check again'),
+      u.newer && u.canInstall ? h('button', {class:'btn', onclick:ask({title:`Update to ${u.latest}?`, danger:false, confirm:'Update now',
+        body:'Sandbar downloads the release, checks it, and restarts. Monitoring and paging pause for a few seconds. Your data is not changed.'}, async () => {
+        await api('POST', V1 + '/system/update', {}); wait(u.current); })}, `Update to ${u.latest}`) : null].filter(Boolean));
+  };
+  const wait = async from => {
+    actions.replaceChildren(); status.textContent = 'Updating.';
+    for (let i = 0; i < 120; i++) {
+      await new Promise(r => setTimeout(r, 1500));
+      let s; try { s = await api('GET', V1 + '/system/update'); } catch (e) { detail.textContent = 'Restarting.'; continue; }
+      if (s.current !== from) { toast(`Updated to ${s.current}.`); return refresh(); }
+      if (s.failed) { toast(s.failed, true); return paint(s); }
+      detail.textContent = s.line || s.stage;
+    }
+    toast('Sandbar did not come back in time. Check the server.', true);
+  };
+  paint(u);
+  return card('Software update', h('div', {class:'stack'}, status, detail, actions));
+}
+
 views.account = async () => {
   const keys = await api('GET', V1 + '/apikeys'), pw = h('input', {type:'password', autocomplete:'new-password'});
+  const upd = S.me.role === 'admin' ? await api('GET', V1 + '/system/update').catch(() => null) : null;
   S.headerAction = null;
   return h('div', {}, card('Password', h('div', {class:'stack'}, field('New password', pw, 'At least 10 characters. You are signed out on other devices.'),
       h('div', {class:'actions form-actions'}, h('button', {class:'btn', onclick:act(async () => { await api('POST', `${V1}/users/${S.me.id}/password`, {password:pw.value}); pw.value = ''; toast('Password changed. Sign in again.'); S.me = null; boot(); })}, 'Change password')))),
@@ -766,7 +794,7 @@ views.account = async () => {
       h('button', {class:'ghost', onclick:() => { const n = h('input'); dialogForm('Create an API key', null, [field('Name', n, 'Where you will use it, such as "prometheus".')], 'Create key', async () => {
         const k = await api('POST', V1 + '/apikeys', {name:n.value}); refresh();
         const done = modal('Copy your API key', h('div', {class:'stack'}, h('p', {class:'dim', style:'margin:0'}, 'This is the only time it is shown.'), cmdBox(k.token), h('div', {class:'actions form-actions'}, h('button', {class:'btn', onclick:() => done()}, 'Done')))); }); }}, icon('add', 16), 'Create API key')),
-    card('Your personal topic', h('p', {class:'dim'}, 'Subscribe to this topic in the ntfy app, or in a browser on the Topics page, to be paged when an escalation policy reaches you.'), cmdBox(S.me.topic)));
+    card('Your personal topic', h('p', {class:'dim'}, 'Subscribe to this topic in the ntfy app, or in a browser on the Topics page, to be paged when an escalation policy reaches you.'), cmdBox(S.me.topic)), upd ? updateCard(upd) : null);
 };
 
 boot();