From 654cf6d0faa9a9d04131cfcf74085a1651038d57 Mon Sep 17 00:00:00 2001 From: Veillax <126523099+Veillax@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:58:23 -0600 Subject: [PATCH] Publish alerts to a Sandbar topic and keep a Sandbar monitor for every resource with health checks A Sandbar topic channel publishes alerts as ntfy-style messages (API key optional). Under Server, Settings, Hammerhead can create, update and remove a Sandbar monitor per enabled resource with health checks, linked by resource id so monitors made by hand are left alone. The key is sealed at rest. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 12 ++ alerts.go | 67 +++++++-- alerts_api.go | 5 +- db.go | 3 +- sandbar.go | 343 +++++++++++++++++++++++++++++++++++++++++++++++ sandbar_test.go | 350 ++++++++++++++++++++++++++++++++++++++++++++++++ server.go | 1 + settings.go | 10 +- store.go | 13 +- web/index.html | 31 +++-- 10 files changed, 810 insertions(+), 25 deletions(-) create mode 100644 sandbar.go create mode 100644 sandbar_test.go diff --git a/README.md b/README.md index ac963ba..b77a2c9 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,18 @@ headers, Host rewrite, HTTPS to the backend, certificate status per port. it, so restarts and blips stay quiet. One message when it starts, one when it clears, optional reminders. Active alerts are stored, so restarting Hammerhead doesn't repeat them, and an alert is only marked resolved once the thing it describes has been re-checked. +- **Sandbar** ([self-hosted notifications, monitors and on-call](https://git.veillax.com/Veillax/sandbar)) can + receive alerts in two ways. A *Sandbar* channel raises the alert through its API (and resolves it when the + problem clears), so the escalation policy you name pages whoever is on call. A *Sandbar topic* channel + publishes a plain message to a topic, which anyone subscribed (the web page, `curl`, the ntfy phone app) sees; + the API key is optional there, for servers that don't ask for one. Keys are sealed at rest and never sent back + to the browser. +- **Sandbar monitors** (Server, Settings): turn this on with Sandbar's address and an administrator's API key, + and Hammerhead creates a Sandbar monitor for every enabled resource that has health checks: a website is + fetched at its public address (a sign-in page counts as up), a TCP port is connected to on the edge's public + IP. Each gets uptime history, an optional place on the status page, and the escalation policy you pick. + Redirects, UDP, wildcard hostnames and resources with health checks off get none. Hammerhead re-syncs every + minute, renames and removes the monitors it created, and never touches ones you made in Sandbar by hand. - A dead edge silences its own ports' alerts (the edge alert already says it). Channels are per organization; **server channels** get edge alerts. - Webhooks can't be pointed at internal addresses (no metadata service, no LAN) unless a server admin diff --git a/alerts.go b/alerts.go index 04cfe65..2a02799 100644 --- a/alerts.go +++ b/alerts.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "net/url" + "regexp" "sort" "strings" "time" @@ -23,21 +24,25 @@ import ( // message when it starts and one when it clears, and optionally a reminder. // Active alerts are kept in the database so a restart doesn't repeat them. +var topicName = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + const ( defaultDebounce = 2 * time.Minute defaultOffline = 90 * time.Second ) -// AlertChannel is somewhere alerts are sent: an email address, a webhook, or a -// Sandbar server (which pages whoever is on call and tracks the alert). +// AlertChannel is somewhere alerts are sent: an email address, a webhook, a +// Sandbar server (which pages whoever is on call and tracks the alert), or a +// Sandbar topic (a plain message for everyone subscribed to it). type AlertChannel struct { ID string `json:"id"` OrgID string `json:"orgId"` // "" = a server-level channel (for edges) Name string `json:"name"` - Kind string `json:"kind"` // "email", "webhook" or "sandbar" + Kind string `json:"kind"` // "email", "webhook", "sandbar" or "topic" Target string `json:"target"` // address(es), a webhook URL, or Sandbar's address (secret: URLs often carry tokens) Format string `json:"format"` // webhook body: "json", "slack", "discord" or "ntfy" - Token string `json:"token"` // Sandbar: API key (secret, never sent back to the browser) + Token string `json:"token"` // Sandbar: API key (secret, never sent back to the browser; optional for a topic) + Topic string `json:"topic"` // topic: the topic to publish to Policy string `json:"policy"` // Sandbar: escalation policy id; blank sends the alert without paging anyone AllowPrivate bool `json:"allowPrivate"` Enabled bool `json:"enabled"` @@ -368,6 +373,8 @@ func (s *Server) send(ch AlertChannel, n alertNote) error { return errors.Join(errs...) case "sandbar": return s.sendSandbar(ch, n) + case "topic": + return s.sendTopic(ch, n) case "webhook": body, ctype, hdr := webhookBody(ch.Format, n) req, err := http.NewRequest("POST", ch.Target, bytes.NewReader(body)) @@ -432,6 +439,38 @@ func (s *Server) sendSandbar(ch AlertChannel, n alertNote) error { "detail": "Organization: " + n.Org + "\nSince: " + n.Since.Format(time.RFC1123), "severity": "critical", "policy": ch.Policy}) } +// sendTopic publishes the alert as a message on a Sandbar topic (ntfy's request +// works too), so everyone subscribed, including the ntfy phone apps, sees it. A +// topic has no alert to close, so a recovery is a second, calmer message. +func (s *Server) sendTopic(ch AlertChannel, n alertNote) error { + prio, tags := "urgent", "rotating_light" + if n.Status != "firing" { + prio, tags = "default", "white_check_mark" + } + req, err := http.NewRequest("POST", strings.TrimRight(ch.Target, "/")+"/"+ch.Topic, strings.NewReader(n.text()+"\n\nOrganization: "+n.Org)) + if err != nil { + return err + } + req.Header.Set("Title", n.subject()) + req.Header.Set("Priority", prio) + req.Header.Set("Tags", tags) + if ch.Token != "" { + req.Header.Set("Authorization", "Bearer "+ch.Token) + } + resp, err := netguard.Client(ch.AllowPrivate).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch { + case resp.StatusCode == 401 || resp.StatusCode == 403: + return errors.New("Sandbar rejected the API key") + case resp.StatusCode >= 300: + return fmt.Errorf("Sandbar answered %s", resp.Status) + } + return nil +} + func webhookBody(format string, n alertNote) (body []byte, ctype string, hdr map[string]string) { text := n.subject() + "\n" + n.text() switch format { @@ -462,7 +501,7 @@ func (s *Server) validateChannel(ch *AlertChannel, trusted bool) error { return errors.New("enter one or more email addresses, separated by commas") } } - ch.Format, ch.Token, ch.Policy, ch.AllowPrivate = "", "", "", false + ch.Format, ch.Token, ch.Policy, ch.Topic, ch.AllowPrivate = "", "", "", "", false case "sandbar": u, err := url.Parse(ch.Target) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { @@ -471,7 +510,19 @@ func (s *Server) validateChannel(ch *AlertChannel, trusted bool) error { if ch.Token = strings.TrimSpace(ch.Token); ch.Token == "" { return errors.New("enter an API key from Sandbar's Account page") } - ch.Policy, ch.Format = strings.TrimSpace(ch.Policy), "" + ch.Policy, ch.Format, ch.Topic = strings.TrimSpace(ch.Policy), "", "" + if ch.AllowPrivate && !trusted { + return errors.New("sending to private addresses (like a service on your home network) needs a server administrator to allow it for this organization") + } + case "topic": + u, err := url.Parse(ch.Target) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return errors.New("enter Sandbar's address, such as https://sandbar.example.com") + } + if ch.Topic = strings.TrimSpace(ch.Topic); !topicName.MatchString(ch.Topic) { + return errors.New("a topic is 1 to 64 letters, digits, - or _") + } + ch.Token, ch.Policy, ch.Format = strings.TrimSpace(ch.Token), "", "" if ch.AllowPrivate && !trusted { return errors.New("sending to private addresses (like a service on your home network) needs a server administrator to allow it for this organization") } @@ -480,7 +531,7 @@ func (s *Server) validateChannel(ch *AlertChannel, trusted bool) error { if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return errors.New("the webhook must be an http:// or https:// URL") } - ch.Token, ch.Policy = "", "" + ch.Token, ch.Policy, ch.Topic = "", "", "" switch ch.Format { case "json", "slack", "discord", "ntfy": case "": @@ -492,7 +543,7 @@ func (s *Server) validateChannel(ch *AlertChannel, trusted bool) error { return errors.New("sending to private addresses (like a service on your home network) needs a server administrator to allow it for this organization") } default: - return errors.New("a channel is an email, a webhook or a Sandbar server") + return errors.New("a channel is an email, a webhook, a Sandbar server or a Sandbar topic") } if ch.Name == "" { ch.Name = ch.Kind diff --git a/alerts_api.go b/alerts_api.go index 656e319..c2fe652 100644 --- a/alerts_api.go +++ b/alerts_api.go @@ -26,6 +26,9 @@ func preview(ch AlertChannel) string { if ch.Kind == "email" { return ch.Target } + if u, err := url.Parse(ch.Target); err == nil && ch.Kind == "topic" { + return u.Scheme + "://" + u.Host + "/" + ch.Topic + } if u, err := url.Parse(ch.Target); err == nil { return u.Scheme + "://" + u.Host + "/…" } @@ -95,7 +98,7 @@ func (s *Server) saveChannel(w http.ResponseWriter, r *http.Request, sc alertSco if in.Target == "" { in.Target = existing.Target // blank keeps the saved URL/address } - if id != "" && in.Token == "" && in.Kind == "sandbar" { + if id != "" && in.Token == "" && (in.Kind == "sandbar" || in.Kind == "topic") { in.Token = existing.Token // blank keeps the saved API key } } diff --git a/db.go b/db.go index ea65bfa..6cc048e 100644 --- a/db.go +++ b/db.go @@ -117,6 +117,7 @@ func (d *DB) sealed(v any) any { return &c case Settings: x.SMTP.Pass = b.Seal(x.SMTP.Pass) + x.Sandbar.Token = b.Seal(x.Sandbar.Token) return x } return v @@ -146,7 +147,7 @@ func (d *DB) unseal(v any) error { case *AlertChannel: return errors.Join(d.open1(&x.Target), d.open1(&x.Token)) case *Settings: - return d.open1(&x.SMTP.Pass) + return errors.Join(d.open1(&x.SMTP.Pass), d.open1(&x.Sandbar.Token)) } return nil } diff --git a/sandbar.go b/sandbar.go new file mode 100644 index 0000000..b57fca4 --- /dev/null +++ b/sandbar.go @@ -0,0 +1,343 @@ +package hammerhead + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "git.veillax.com/Veillax/reef/model" + "git.veillax.com/Veillax/reef/netguard" +) + +// Sandbar monitors: with this switched on under Server, Settings, every enabled +// resource that has health checks gets a monitor on a Sandbar server, which then +// tracks its uptime, shows it on the status page and pages whoever is on call. +// Sandbar probes the public address (so it sees what visitors see), where +// Hammerhead's own health checks look at the targets behind it. Hammerhead only +// ever changes the monitors it created: the link from resource to monitor is +// kept in the settings, so a monitor you add in Sandbar by hand is never touched. + +// SandbarSync is the connection to Sandbar for creating monitors. +type SandbarSync struct { + Enabled bool `json:"enabled"` + URL string `json:"url"` + Token string `json:"token"` // an administrator's API key (secret, never sent back to the browser) + PolicyID string `json:"policyId"` // escalation policy for the alerts the monitors raise; blank pages nobody + Public bool `json:"public"` // show the monitors on Sandbar's status page + AllowPrivate bool `json:"allowPrivate"` // Sandbar is on a private network, and so are the edge addresses it will probe + Monitors map[string]string `json:"monitors,omitempty"` // resource id -> Sandbar monitor id +} + +// sandbarSyncStatus is how the last sync went, for the settings page. +type sandbarSyncStatus struct { + At time.Time `json:"at"` + Error string `json:"error"` + Monitors int `json:"monitors"` +} + +const sandbarSyncEvery = time.Minute + +type monitorSpec struct { + Name string + Kind string + Target string + IntervalS int + Expect string +} + +// edgeHost is the address visitors reach a resource's edge on: the address it is +// bound to, else the first public address the edge reported, else the host of its +// tunnel endpoint. +func edgeHost(e *model.Edge, bind string) string { + if ip := net.ParseIP(bind); ip != nil && !ip.IsUnspecified() { + return bind + } + var first string + for _, a := range e.IPs { + ip := net.ParseIP(a) + if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() { + continue + } + if !ip.IsPrivate() { + return a + } + if first == "" { + first = a + } + } + if first != "" { + return first + } + host, _, _ := net.SplitHostPort(e.TunnelAddr) + return host +} + +// monitorFor describes the Sandbar monitor a resource should have, or false when +// it has nothing worth probing (a redirect, UDP, health checks off, a wildcard). +func (s *Server) monitorFor(m *model.Mapping) (monitorSpec, bool) { + if !m.Enabled || m.HealthSeconds == -1 || m.RedirectTo != "" || m.Proto == model.ProtoUDP { + return monitorSpec{}, false + } + e := s.findEdge(m.EdgeID) + if e == nil { + return monitorSpec{}, false + } + spec := monitorSpec{Name: m.Name, IntervalS: 30} + if m.HealthSeconds > 0 { + spec.IntervalS = max(m.HealthSeconds, 10) + } + host := m.Hostname + if strings.HasPrefix(host, "*") { + return monitorSpec{}, false + } + if host == "" { + if host = edgeHost(e, m.BindIP); host == "" { + return monitorSpec{}, false + } + } + addr := net.JoinHostPort(host, strconv.Itoa(m.Port)) + switch m.Proto { + case model.ProtoHTTP, model.ProtoHTTPS: + u := url.URL{Scheme: m.Proto, Host: addr, Path: m.HealthPath} + if (m.Proto == model.ProtoHTTP && m.Port == 80) || (m.Proto == model.ProtoHTTPS && m.Port == 443) { + u.Host = strings.Trim(host, "[]") // a bare IPv6 address needs its brackets back + if strings.Contains(host, ":") { + u.Host = "[" + u.Host + "]" + } + } + spec.Kind, spec.Target = "http", u.String() + if m.Auth.Protected() { + spec.Expect = "200-399" // visitors are sent to a sign-in page, which is the site working + } + default: + spec.Kind, spec.Target = "tcp", addr + } + return spec, true +} + +// desiredMonitorsLocked is the monitor every qualifying resource should have, by resource id. +func (s *Server) desiredMonitorsLocked() map[string]monitorSpec { + out := map[string]monitorSpec{} + for _, m := range s.st.Mappings { + if spec, ok := s.monitorFor(m); ok { + out[m.ID] = spec + } + } + return out +} + +func sandbarCall(cfg SandbarSync, method, path string, body any, out any) error { + var rd io.Reader + if body != nil { + raw, _ := json.Marshal(body) + rd = bytes.NewReader(raw) + } + req, err := http.NewRequest(method, strings.TrimRight(cfg.URL, "/")+path, rd) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+cfg.Token) + resp, err := netguard.Client(cfg.AllowPrivate).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch { + case resp.StatusCode == 401: + return errors.New("Sandbar rejected the API key") + case resp.StatusCode == 403: + return errors.New("Sandbar needs an administrator's API key to manage monitors") + case resp.StatusCode == 404 && method == "DELETE": + return nil // already gone + case resp.StatusCode >= 300: + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300)) + return fmt.Errorf("Sandbar answered %s %s", resp.Status, strings.TrimSpace(string(msg))) + } + if out != nil { + return json.NewDecoder(resp.Body).Decode(out) + } + return nil +} + +// syncSandbar makes Sandbar's monitors match the resources: it creates the ones +// that are missing, corrects the ones that drifted, and removes those whose +// resource is gone or no longer has health checks. +func (s *Server) syncSandbar() (err error) { + s.sandbarMu.Lock() + defer s.sandbarMu.Unlock() + s.mu.Lock() + cfg := s.st.Settings.Sandbar + if !cfg.Enabled { + s.mu.Unlock() + return nil + } + want := s.desiredMonitorsLocked() + s.mu.Unlock() + links := map[string]string{} + for k, v := range cfg.Monitors { + links[k] = v + } + defer func() { + st := sandbarSyncStatus{At: time.Now(), Monitors: len(links)} + if err != nil { + st.Error = err.Error() + } + s.mu.Lock() + s.sandbarStatus = st + if now := &s.st.Settings.Sandbar; now.URL == cfg.URL && !sameLinks(now.Monitors, links) { + now.Monitors = links // kept even after a failure: a monitor already created must not be created twice + s.save() + } + s.mu.Unlock() + }() + + var remote []map[string]any + if err := sandbarCall(cfg, "GET", "/api/v1/monitors", nil, &remote); err != nil { + return err + } + byID := map[string]map[string]any{} + for _, m := range remote { + if id, _ := m["id"].(string); id != "" { + byID[id] = m + } + } + + ids := make([]string, 0, len(want)) + for id := range want { + ids = append(ids, id) + } + sort.Strings(ids) + var errs []error + for _, id := range ids { + spec := want[id] + fields := map[string]any{"name": spec.Name, "kind": spec.Kind, "target": spec.Target, "intervalS": spec.IntervalS, "expect": spec.Expect, + "policyId": cfg.PolicyID, "public": cfg.Public, "allowPrivate": cfg.AllowPrivate, "enabled": true} + cur, exists := byID[links[id]] + if !exists { + var made struct{ ID string } + if err := sandbarCall(cfg, "POST", "/api/v1/monitors", fields, &made); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", spec.Name, err)) + continue + } + links[id] = made.ID + continue + } + drift := false + for k, v := range fields { + if !sameJSON(cur[k], v) { + cur[k], drift = v, true + } + } + if drift { // the whole document is replaced, so what Sandbar added (token, retries) rides along + if err := sandbarCall(cfg, "PUT", "/api/v1/monitors/"+links[id], cur, nil); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", spec.Name, err)) + } + } + } + for id, rid := range links { + if _, still := want[id]; still { + continue + } + if err := sandbarCall(cfg, "DELETE", "/api/v1/monitors/"+rid, nil, nil); err != nil { + errs = append(errs, err) + continue + } + delete(links, id) + } + return errors.Join(errs...) +} + +func sameJSON(a, b any) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + if string(x) == "null" { // a field Sandbar left out is its zero value + x = []byte(`""`) + if _, isNum := b.(int); isNum { + x = []byte("0") + } else if _, isBool := b.(bool); isBool { + x = []byte("false") + } + } + return bytes.Equal(x, y) +} + +func sameLinks(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func (s *Server) sandbarLoop() { + t := time.NewTicker(sandbarSyncEvery) + defer t.Stop() + for { + select { + case <-s.stop: + return + case <-t.C: + if err := s.syncSandbar(); err != nil { + log.Printf("sandbar monitors: %v", err) + } + } + } +} + +// syncSandbarNow is the "Sync now" button: the same sync, with its outcome shown. +func (s *Server) syncSandbarNow(w http.ResponseWriter, r *http.Request, p *principal) { + s.mu.Lock() + on := s.st.Settings.Sandbar.Enabled + s.mu.Unlock() + if !on { + fail(w, 400, errors.New("Turn on Sandbar monitors and save the settings first.")) + return + } + if err := s.syncSandbar(); err != nil { + fail(w, 400, err) + return + } + s.mu.Lock() + st := s.sandbarStatus + s.mu.Unlock() + writeJSON(w, 200, st) +} + +// cleanSandbar validates the settings in `in`, filling in what a blank field means. +// Caller holds s.mu. +func (s *Server) cleanSandbar(in *SandbarSync) error { + old := s.st.Settings.Sandbar + in.URL, in.PolicyID = strings.TrimRight(strings.TrimSpace(in.URL), "/"), strings.TrimSpace(in.PolicyID) + if in.Token = strings.TrimSpace(in.Token); in.Token == "" { + in.Token = old.Token // blank keeps the saved API key + } + in.Monitors = old.Monitors // not editable here + if in.URL != old.URL { + in.Monitors = nil // those belong to the previous server + } + if !in.Enabled { + return nil + } + if u, err := url.Parse(in.URL); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return errors.New("enter Sandbar's address, such as https://sandbar.example.com") + } + if in.Token == "" { + return errors.New("enter an administrator's API key from Sandbar's Account page") + } + return nil +} diff --git a/sandbar_test.go b/sandbar_test.go new file mode 100644 index 0000000..0d94b02 --- /dev/null +++ b/sandbar_test.go @@ -0,0 +1,350 @@ +package hammerhead + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "git.veillax.com/Veillax/reef/model" +) + +// fakeSandbar keeps monitors in memory the way Sandbar's /api/v1/monitors does. +type fakeSandbar struct { + URL string + mu sync.Mutex + mon map[string]map[string]any + n int + // requests that carried the wrong key are counted, not served + badAuth int + calls []string +} + +func newFakeSandbar(t *testing.T) *fakeSandbar { + f := &fakeSandbar{mon: map[string]map[string]any{}} + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if r.Header.Get("Authorization") != "Bearer sb_admin" { + f.badAuth++ + http.Error(w, "no", 401) + return + } + f.calls = append(f.calls, r.Method) + id := strings.TrimPrefix(r.URL.Path, "/api/v1/monitors/") + switch { + case r.Method == "GET" && r.URL.Path == "/api/v1/monitors": + out := []any{} + for _, m := range f.mon { + out = append(out, m) + } + json.NewEncoder(w).Encode(out) + case r.Method == "POST" && r.URL.Path == "/api/v1/monitors": + var m map[string]any + json.NewDecoder(r.Body).Decode(&m) + f.n++ + m["id"], m["retries"] = "mon"+strconv.Itoa(f.n), 2 // Sandbar adds its own fields + f.mon[m["id"].(string)] = m + json.NewEncoder(w).Encode(m) + case r.Method == "PUT" && f.mon[id] != nil: + var m map[string]any + json.NewDecoder(r.Body).Decode(&m) + f.mon[id] = m + json.NewEncoder(w).Encode(m) + case r.Method == "DELETE": + delete(f.mon, id) + w.Write([]byte(`{"ok":true}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(ts.Close) + f.URL = ts.URL + return f +} + +func (f *fakeSandbar) names() map[string]string { // monitor id -> "kind target" + f.mu.Lock() + defer f.mu.Unlock() + out := map[string]string{} + for id, m := range f.mon { + out[id] = m["kind"].(string) + " " + m["target"].(string) + } + return out +} + +func (f *fakeSandbar) count(method string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, c := range f.calls { + if c == method { + n++ + } + } + return n +} + +func enableSandbarSync(f *fx, sb *fakeSandbar) { + f.s.mu.Lock() + f.s.st.Edges[0].IPs = []string{"10.0.0.5", "203.0.113.7"} + f.s.st.Settings.Sandbar = SandbarSync{Enabled: true, URL: sb.URL, Token: "sb_admin", PolicyID: "pol1", Public: true, AllowPrivate: true} + f.s.mu.Unlock() +} + +func TestSandbarMonitorsFollowTheResources(t *testing.T) { + f := newFx(t) + sb := newFakeSandbar(t) + enableSandbarSync(f, sb) + f.s.mu.Lock() + f.s.st.Mappings = append(f.s.st.Mappings, + &model.Mapping{ID: "m2", OrgID: f.org, Name: "Blog", EdgeID: "e1", Proto: "https", Port: 443, Hostname: "blog.example.com", HealthPath: "/ping", Enabled: true, + Targets: []model.Target{{SiteID: "s1", Host: "192.168.1.31", Port: 80}}}, + &model.Mapping{ID: "m3", OrgID: f.org, Name: "Docs", EdgeID: "e1", Proto: "https", Port: 8443, Hostname: "docs.example.com", Enabled: true, Auth: model.Auth{Password: true}, + Targets: []model.Target{{SiteID: "s1", Host: "192.168.1.32", Port: 80}}}, + &model.Mapping{ID: "m4", OrgID: f.org, Name: "Unchecked", EdgeID: "e1", Proto: "tcp", Port: 2222, HealthSeconds: -1, Enabled: true}, + &model.Mapping{ID: "m5", OrgID: f.org, Name: "Voice", EdgeID: "e1", Proto: "udp", Port: 9987, Enabled: true}, + &model.Mapping{ID: "m6", OrgID: f.org, Name: "Wildcard", EdgeID: "e1", Proto: "https", Port: 443, Hostname: "*.example.com", Enabled: true}, + &model.Mapping{ID: "m7", OrgID: f.org, Name: "Off", EdgeID: "e1", Proto: "tcp", Port: 3333}, + ) + f.s.mu.Unlock() + + if err := f.s.syncSandbar(); err != nil { + t.Fatal(err) + } + got := sb.names() + want := map[string]bool{"tcp 203.0.113.7:25565": true, "http https://blog.example.com/ping": true, "http https://docs.example.com:8443": true} + if len(got) != len(want) { + t.Fatalf("monitors created: %v", got) + } + for _, v := range got { + if !want[v] { + t.Errorf("unexpected monitor %q", v) + } + } + for _, m := range sb.mon { + if m["policyId"] != "pol1" || m["public"] != true || m["enabled"] != true { + t.Errorf("monitor settings not applied: %v", m) + } + if m["name"] == "Docs" && m["expect"] != "200-399" { + t.Errorf("a protected site should accept its sign-in redirect: %v", m) + } + } + + // A second pass changes nothing. + posts := sb.count("POST") + if err := f.s.syncSandbar(); err != nil { + t.Fatal(err) + } + if sb.count("POST") != posts || sb.count("PUT") != 0 || sb.count("DELETE") != 0 { + t.Fatalf("a settled sync still wrote: %v", sb.calls) + } + + // A rename and a changed policy are pushed, and Sandbar's own fields survive. + f.s.mu.Lock() + f.s.findMapping("m1").Name = "Minecraft Java" + f.s.st.Settings.Sandbar.PolicyID = "pol2" + f.s.mu.Unlock() + if err := f.s.syncSandbar(); err != nil { + t.Fatal(err) + } + var renamed bool + for _, m := range sb.mon { + if m["policyId"] != "pol2" || m["retries"] != float64(2) { + t.Errorf("update lost data or missed the policy: %v", m) + } + renamed = renamed || m["name"] == "Minecraft Java" + } + if !renamed { + t.Error("the rename was not pushed") + } + + // Deleting a resource, or turning its checks off, removes its monitor; one added by hand stays. + sb.mu.Lock() + sb.mon["hand"] = map[string]any{"id": "hand", "name": "mine", "kind": "tcp", "target": "x:1"} + sb.mu.Unlock() + f.s.mu.Lock() + f.s.findMapping("m2").HealthSeconds = -1 + f.s.st.Mappings = filter(f.s.st.Mappings, func(m *model.Mapping) bool { return m.ID != "m3" }) + f.s.mu.Unlock() + if err := f.s.syncSandbar(); err != nil { + t.Fatal(err) + } + got = sb.names() + if len(got) != 2 || got["hand"] == "" { + t.Fatalf("after removing resources: %v", got) + } + f.s.mu.Lock() + links := len(f.s.st.Settings.Sandbar.Monitors) + f.s.mu.Unlock() + if links != 1 { + t.Errorf("links kept for %d resources, want 1", links) + } +} + +func TestSandbarSyncRecreatesAMonitorDeletedInSandbarAndSurvivesARestart(t *testing.T) { + f := newFx(t) + sb := newFakeSandbar(t) + enableSandbarSync(f, sb) + if err := f.s.syncSandbar(); err != nil { + t.Fatal(err) + } + sb.mu.Lock() + sb.mon = map[string]map[string]any{} + sb.mu.Unlock() + if err := f.s.syncSandbar(); err != nil { + t.Fatal(err) + } + if len(sb.names()) != 1 { + t.Fatalf("not recreated: %v", sb.names()) + } + + // the links are stored, so a restart does not create a second monitor + f.s.mu.Lock() + err := f.s.save() + f.s.mu.Unlock() + if err != nil { + t.Fatal(err) + } + dir, key := f.s.dir, f.s.opts.MasterKey + f.s.Close() + s2, err := New(dir, Options{MasterKey: key, AlertInterval: time.Hour}) + if err != nil { + t.Fatal(err) + } + defer s2.Close() + if err := s2.syncSandbar(); err != nil { + t.Fatal(err) + } + if len(sb.names()) != 1 { + t.Fatalf("a restart duplicated the monitor: %v", sb.names()) + } +} + +func TestSandbarSyncReportsABadKeyAndIsOffByDefault(t *testing.T) { + f := newFx(t) + sb := newFakeSandbar(t) + if err := f.s.syncSandbar(); err != nil || sb.badAuth != 0 || len(sb.calls) != 0 { + t.Fatal("sync ran while switched off") + } + enableSandbarSync(f, sb) + f.s.mu.Lock() + f.s.st.Settings.Sandbar.Token = "wrong" + f.s.mu.Unlock() + if err := f.s.syncSandbar(); err == nil || !strings.Contains(err.Error(), "API key") { + t.Fatalf("error: %v", err) + } + f.s.mu.Lock() + msg := f.s.sandbarStatus.Error + f.s.mu.Unlock() + if msg == "" { + t.Error("the failure was not recorded for the settings page") + } +} + +func TestSandbarSettingsKeepTheKeySealedAndHidden(t *testing.T) { + f := newFx(t) + f.s.mu.Lock() + in := SandbarSync{Enabled: true, URL: " https://sandbar.example.com/ ", Token: " sb_admin "} + err := f.s.cleanSandbar(&in) + f.s.st.Settings.Sandbar = in + f.s.save() + f.s.mu.Unlock() + if err != nil || in.URL != "https://sandbar.example.com" || in.Token != "sb_admin" { + t.Fatalf("%v %+v", err, in) + } + + // a blank key in a later save keeps the saved one; a new address forgets the old server's links + f.s.mu.Lock() + f.s.st.Settings.Sandbar.Monitors = map[string]string{"m1": "mon1"} + next := SandbarSync{Enabled: true, URL: "https://other.example.com"} + err = f.s.cleanSandbar(&next) + f.s.mu.Unlock() + if err != nil || next.Token != "sb_admin" || next.Monitors != nil { + t.Fatalf("%v %+v", err, next) + } + for _, bad := range []SandbarSync{{Enabled: true, URL: "not a url", Token: "k"}, {Enabled: true, URL: "https://x.example.com", Token: ""}} { + bad := bad + f.s.mu.Lock() + f.s.st.Settings.Sandbar.Token = "" + err := f.s.cleanSandbar(&bad) + f.s.mu.Unlock() + if err == nil { + t.Errorf("accepted %+v", bad) + } + } + + // at rest the key is encrypted, and the settings API never returns it + raw, _ := json.Marshal(f.s.db.sealed(f.s.st.Settings)) + if strings.Contains(string(raw), "sb_admin") { + t.Error("the API key is stored in the clear") + } + f.s.mu.Lock() + f.s.st.Settings.Sandbar.Token = "sb_admin" + f.s.mu.Unlock() + rec := httptest.NewRecorder() + f.s.getSettings(rec, httptest.NewRequest("GET", "/api/admin/settings", nil), nil) + if strings.Contains(rec.Body.String(), "sb_admin") || !strings.Contains(rec.Body.String(), `"sandbarTokenSet":true`) { + t.Errorf("settings response: %s", rec.Body.String()) + } +} + +func TestTopicChannelPublishes(t *testing.T) { + f := newFx(t) + var gotPath, gotTitle, gotPrio, gotAuth, gotBody string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b := make([]byte, 512) + n, _ := r.Body.Read(b) + gotPath, gotTitle, gotPrio, gotAuth, gotBody = r.URL.Path, r.Header.Get("Title"), r.Header.Get("Priority"), r.Header.Get("Authorization"), string(b[:n]) + })) + defer ts.Close() + ch := AlertChannel{ID: "t1", Kind: "topic", Target: ts.URL + "/", Topic: "ops", Token: "sb_k", AllowPrivate: true, Enabled: true} + n := alertNote{Status: "firing", Org: "Acme", Name: "Minecraft", Message: "Minecraft can't listen", Since: time.Now()} + if err := f.s.send(ch, n); err != nil { + t.Fatal(err) + } + if gotPath != "/ops" || gotTitle != "[Hammerhead] Problem: Minecraft" || gotPrio != "urgent" || gotAuth != "Bearer sb_k" || !strings.Contains(gotBody, "can't listen") { + t.Errorf("published %q %q %q %q %q", gotPath, gotTitle, gotPrio, gotAuth, gotBody) + } + n.Status, ch.Token = "resolved", "" + if err := f.s.send(ch, n); err != nil { + t.Fatal(err) + } + if gotPrio != "default" || gotAuth != "" || !strings.HasPrefix(gotBody, "Resolved") { + t.Errorf("recovery %q %q %q", gotPrio, gotAuth, gotBody) + } + + // a refusal surfaces as an error + deny := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "no", 401) })) + defer deny.Close() + ch.Target = deny.URL + if err := f.s.send(ch, n); err == nil || !strings.Contains(err.Error(), "API key") { + t.Errorf("error: %v", err) + } +} + +func TestTopicChannelValidation(t *testing.T) { + f := newFx(t) + for _, ch := range []AlertChannel{ + {Kind: "topic", Target: "not a url", Topic: "ops"}, + {Kind: "topic", Target: "https://sandbar.example.com", Topic: ""}, + {Kind: "topic", Target: "https://sandbar.example.com", Topic: "has space"}, + {Kind: "topic", Target: "https://sandbar.example.com", Topic: "ops", AllowPrivate: true}, // needs a trusted caller + } { + ch := ch + if err := f.s.validateChannel(&ch, false); err == nil { + t.Fatalf("accepted %+v", ch) + } + } + ok := AlertChannel{Kind: "topic", Target: "https://sandbar.example.com", Topic: " ops ", Token: " ", Policy: "x", Format: "slack"} + if err := f.s.validateChannel(&ok, false); err != nil || ok.Topic != "ops" || ok.Token != "" || ok.Policy != "" || ok.Format != "" { + t.Fatalf("%v %+v", err, ok) + } + if p := preview(ok); p != "https://sandbar.example.com/ops" { + t.Errorf("preview %q", p) + } +} diff --git a/server.go b/server.go index 2b60aae..a4589bf 100644 --- a/server.go +++ b/server.go @@ -97,6 +97,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /api/admin/settings", s.withServerAdmin(s.getSettings)) mux.HandleFunc("PUT /api/admin/settings", s.withServerAdmin(s.putSettings)) mux.HandleFunc("POST /api/admin/settings/test-mail", s.withServerAdmin(s.testMail)) + mux.HandleFunc("POST /api/admin/settings/sandbar-sync", s.withServerAdmin(s.syncSandbarNow)) mux.HandleFunc("GET /api/admin/update", s.withServerAdmin(s.checkUpdate)) mux.HandleFunc("POST /api/admin/update/apply", s.withServerAdmin(s.applyUpdate)) mux.HandleFunc("GET /api/admin/update/status", s.withServerAdmin(s.updateStatus)) diff --git a/settings.go b/settings.go index 7210c2f..c924fac 100644 --- a/settings.go +++ b/settings.go @@ -16,8 +16,9 @@ func (s *Server) getSettings(w http.ResponseWriter, r *http.Request, p *principa s.mu.Lock() defer s.mu.Unlock() st := s.st.Settings - st.SMTP.Pass = "" // write-only - writeJSON(w, 200, map[string]any{"settings": st, "smtpPassSet": s.st.Settings.SMTP.Pass != ""}) + st.SMTP.Pass = "" // write-only + st.Sandbar.Token, st.Sandbar.Monitors = "", nil // write-only + writeJSON(w, 200, map[string]any{"settings": st, "smtpPassSet": s.st.Settings.SMTP.Pass != "", "sandbarTokenSet": s.st.Settings.Sandbar.Token != "", "sandbarSync": s.sandbarStatus}) } func (s *Server) putSettings(w http.ResponseWriter, r *http.Request, p *principal) { @@ -38,6 +39,11 @@ func (s *Server) putSettings(w http.ResponseWriter, r *http.Request, p *principa if in.SMTP.Pass == "" { in.SMTP.Pass = s.st.Settings.SMTP.Pass // blank keeps the saved password } + if err := s.cleanSandbar(&in.Sandbar); err != nil { + s.mu.Unlock() + fail(w, 400, err) + return + } if s.opts.DashboardURL != "" { in.DashboardURL = s.st.Settings.DashboardURL // pinned by flag/env } diff --git a/store.go b/store.go index 6a99b1c..7c44b56 100644 --- a/store.go +++ b/store.go @@ -109,10 +109,11 @@ type SMTP struct { } type Settings struct { - DashboardURL string `json:"dashboardUrl"` - AcmeEmail string `json:"acmeEmail"` - AcmeStaging bool `json:"acmeStaging"` - SMTP SMTP `json:"smtp"` + DashboardURL string `json:"dashboardUrl"` + AcmeEmail string `json:"acmeEmail"` + AcmeStaging bool `json:"acmeStaging"` + SMTP SMTP `json:"smtp"` + Sandbar SandbarSync `json:"sandbar"` // SelfEdgeID is this server's own edge (see HAMMERHEAD_SELF_DOMAIN). SelfEdgeID string `json:"selfEdgeId,omitempty"` } @@ -218,6 +219,9 @@ type Server struct { verAt time.Time mtaSent map[string]time.Time // reminder throttle for repeat notifications + sandbarMu sync.Mutex // one monitor sync at a time (see sandbar.go) + sandbarStatus sandbarSyncStatus // how the last one went + logs *logStore db *DB stop chan struct{} @@ -299,6 +303,7 @@ func New(dir string, opts Options) (*Server, error) { s.loadAlerts() go s.housekeeping() go s.alertLoop() + go s.sandbarLoop() return s, nil } diff --git a/web/index.html b/web/index.html index 8b70d41..bb9ad60 100644 --- a/web/index.html +++ b/web/index.html @@ -1140,32 +1140,36 @@ const FORMATS = [['json', 'JSON'], ['slack', 'Slack'], ['discord', 'Discord'], [ function channelForm(base, existing, trusted) { const c = existing || {kind:'email', format:'json', enabled:true}; const name = h('input', {value:c.name || '', placeholder:'e.g. On-call'}); - const kind = h('select', {}, [['email', 'Email'], ['sandbar', 'Sandbar'], ['webhook', 'Webhook']].map(([v, t]) => h('option', {value:v, selected:v === c.kind}, t))); + const kind = h('select', {}, [['email', 'Email'], ['sandbar', 'Sandbar (pages on-call)'], ['topic', 'Sandbar topic'], ['webhook', 'Webhook']].map(([v, t]) => h('option', {value:v, selected:v === c.kind}, t))); const target = h('input', {value:c.kind === 'email' ? c.target || '' : '', placeholder:c.kind === 'webhook' && existing ? 'unchanged' : ''}); const token = h('input', {type:'password', autocomplete:'off', placeholder:existing && c.kind === 'sandbar' ? 'unchanged' : 'sb_…'}); const policy = h('input', {value:c.policy || '', placeholder:'Optional'}); + const topic = h('input', {value:c.topic || '', placeholder:'e.g. alerts'}); + const topicToken = h('input', {type:'password', autocomplete:'off', placeholder:existing && c.kind === 'topic' ? 'unchanged' : 'Optional'}); const sandbarBox = h('div', {class:'grid'}, field('API key', token, 'Create one on Sandbar\'s Account page.'), field('Escalation policy ID', policy, 'Who gets paged. Leave blank to send alerts without paging anyone.')); + const topicBox = h('div', {class:'grid'}, field('Topic', topic, 'Messages are published here, so anyone subscribed (including the ntfy app) sees them.'), field('API key (optional)', topicToken, 'Leave blank for a server that lets anyone publish. Sandbar itself asks for a key.')); const fmt = h('select', {}, FORMATS.map(([v, t]) => h('option', {value:v, selected:v === (c.format || 'json')}, t))); const priv = chk('This address is on a private network (like a home server)', c.allowPrivate); const en = chk('Enabled', c.enabled !== false); const fmtBox = field('Webhook format', fmt, 'Slack and Discord incoming-webhook URLs work as they are. ntfy sends a push notification.'); const targetBox = field('Where', target); const redraw = () => { - const w = kind.value === 'webhook', sb = kind.value === 'sandbar'; + const w = kind.value === 'webhook', sb = kind.value === 'sandbar' || kind.value === 'topic'; fmtBox.style.display = w ? '' : 'none'; - sandbarBox.style.display = sb ? '' : 'none'; + sandbarBox.style.display = kind.value === 'sandbar' ? '' : 'none'; + topicBox.style.display = kind.value === 'topic' ? '' : 'none'; priv.style.display = (w || sb) && trusted ? '' : 'none'; targetBox.firstChild.textContent =sb ? 'Sandbar address' : 'Where'; - target.placeholder = sb ? (existing && c.kind === 'sandbar' ? 'unchanged' : 'https://sandbar.example.com') : + target.placeholder = sb ? (existing && c.kind === kind.value ? 'unchanged' : 'https://sandbar.example.com') : w ? (existing && c.kind === 'webhook' ? 'unchanged (it may contain a secret token)' : 'https://hooks.example.com/…') : 'you@example.com, oncall@example.com'; }; kind.onchange = redraw; redraw(); const save = act(async () => { - await api(existing ? 'PUT' : 'POST', base + '/channels' + (existing ? '/' + existing.id : ''), {name:name.value, kind:kind.value, target:target.value, format:kind.value === 'webhook' ? fmt.value : '', token:token.value, policy:policy.value, allowPrivate:trusted && priv.box.checked, enabled:en.box.checked}); + await api(existing ? 'PUT' : 'POST', base + '/channels' + (existing ? '/' + existing.id : ''), {name:name.value, kind:kind.value, target:target.value, format:kind.value === 'webhook' ? fmt.value : '', token:kind.value === 'topic' ? topicToken.value : token.value, policy:policy.value, topic:topic.value, allowPrivate:trusted && priv.box.checked, enabled:en.box.checked}); close(); toast(existing ? 'Alert channel saved' : 'Alert channel added'); refresh(); }); const close = modal(existing ? 'Edit channel' : 'Add an alert channel', h('div', {class:'stack'}, - h('div', {class:'grid'}, field('Name', name), field('Type', kind)), targetBox, sandbarBox, fmtBox, priv, en, + h('div', {class:'grid'}, field('Name', name), field('Type', kind)), targetBox, sandbarBox, topicBox, fmtBox, priv, en, h('div', {class:'actions form-actions'}, h('button', {class:'ghost', onclick:() => close()}, 'Cancel'), h('button', {class:'btn', onclick:save}, 'Save'))), {sticky:true}); } @@ -1173,7 +1177,7 @@ async function alertsView(base, scoped, trusted) { const d = await api('GET', base); const channels = h('table', {}, h('thead', {}, h('tr', {}, ['Name', 'Type', 'Sends to', ''].map(x => h('th', {}, x)))), h('tbody', {}, d.channels.map(c => h('tr', {}, h('td', {}, c.name, c.enabled ? null : h('span', {class:'tag', style:'margin-left:6px'}, 'off')), - h('td', {}, c.kind === 'email' ? 'Email' : c.kind === 'sandbar' ? 'Sandbar' : 'Webhook (' + c.format + ')'), h('td', {class:'mono small'}, c.preview), + h('td', {}, c.kind === 'email' ? 'Email' : c.kind === 'sandbar' ? 'Sandbar' : c.kind === 'topic' ? 'Sandbar topic' : 'Webhook (' + c.format + ')'), h('td', {class:'mono small'}, c.preview), h('td', {}, h('div', {class:'actions'}, h('button', {class:'ghost', onclick: act(async () => { await api('POST', base + '/channels/' + c.id + '/test'); toast('Test sent'); })}, 'Send test'), h('button', {class:'ghost', onclick:() => channelForm(base, c, trusted)}, 'Edit'), @@ -1354,13 +1358,19 @@ serverViews.idps = async () => { }; serverViews.settings = async () => { - const { settings: s, smtpPassSet } = await api('GET', '/api/admin/settings'); + const { settings: s, smtpPassSet, sandbarTokenSet, sandbarSync } = await api('GET', '/api/admin/settings'); const f = {url:h('input', {value:s.dashboardUrl, placeholder:'https://hammerhead.example.com'}), acme:h('input', {type:'email', value:s.acmeEmail, placeholder:'you@example.com'}), host:h('input', {value:s.smtp.host}), port:h('input', {type:'number', value:s.smtp.port || ''}), user:h('input', {value:s.smtp.user}), pass:h('input', {type:'password', placeholder:smtpPassSet ? 'unchanged' : ''}), from:h('input', {value:s.smtp.from, placeholder:'Hammerhead '}), mode:h('select', {}, [['starttls', 'STARTTLS'], ['tls', 'TLS'], ['none', 'None']].map(([m, l]) => h('option', {value:m, selected:(s.smtp.mode || 'starttls') === m}, l)))}; const staging = chk('Use the Let\'s Encrypt staging server (untrusted test certificates)', s.acmeStaging); + const sb = s.sandbar || {}; + const sbOn = chk('Create a Sandbar monitor for every resource that has health checks', sb.enabled); + const sbf = {url:h('input', {value:sb.url || '', placeholder:'https://sandbar.example.com'}), key:h('input', {type:'password', autocomplete:'off', placeholder:sandbarTokenSet ? 'unchanged' : 'sb_…'}), policy:h('input', {value:sb.policyId || '', placeholder:'Optional'})}; + const sbPublic = chk('Show the monitors on the status page', sb.public), sbPriv = chk('Sandbar and the edge addresses are on a private network', sb.allowPrivate); + const sbOut = h('div', {class:'dim'}, sandbarSync && sandbarSync.at && sandbarSync.at.slice(0, 4) !== '0001' ? (sandbarSync.error ? 'The last sync failed: ' + sandbarSync.error : 'Last synced ' + ago(sandbarSync.at) + ', ' + sandbarSync.monitors + ' monitors.') : 'Not synced yet.'); const save = act(async () => { await api('PUT', '/api/admin/settings', {dashboardUrl:f.url.value, acmeEmail:f.acme.value, acmeStaging:staging.box.checked, - smtp:{host:f.host.value, port:+f.port.value, user:f.user.value, pass:f.pass.value, from:f.from.value, mode:f.mode.value}}); S.me = await api('GET', '/api/me'); toast('Settings saved'); }); + smtp:{host:f.host.value, port:+f.port.value, user:f.user.value, pass:f.pass.value, from:f.from.value, mode:f.mode.value}, + sandbar:{enabled:sbOn.box.checked, url:sbf.url.value, token:sbf.key.value, policyId:sbf.policy.value, public:sbPublic.box.checked, allowPrivate:sbPriv.box.checked}}); S.me = await api('GET', '/api/me'); toast('Settings saved'); }); const updOut = h('div', {class:'dim'}, 'Not checked yet.'); const updNow = h('button', {class:'ghost', style:'display:none', onclick: act(async () => { await api('POST', '/api/admin/update/apply'); @@ -1388,6 +1398,9 @@ serverViews.settings = async () => { card('Email', h('p', {class:'dim'}, 'Needed for invitation emails and one-time sign-in codes.'), h('div', {class:'grid'}, field('SMTP host', f.host), field('Port', f.port), field('Security', f.mode), field('Username', f.user), field('Password', f.pass), field('From', f.from)), h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick: act(async () => { await save(); await api('POST', '/api/admin/settings/test-mail'); toast('Test email sent to ' + S.me.email); })}, 'Save and send a test email'))), + card('Sandbar monitors', h('p', {class:'dim'}, 'Hammerhead adds a monitor to Sandbar for each enabled resource with health checks (a website is fetched at its address, a port is connected to), so it gets uptime history, a status page and on-call paging. Monitors you add in Sandbar yourself are never touched. Hammerhead keeps them up to date every minute and removes the ones it created when a resource is deleted or its checks are turned off.'), + sbOn, h('div', {class:'grid'}, field('Sandbar address', sbf.url), field('Administrator API key', sbf.key, 'Create one on the Account page in Sandbar. Managing monitors needs an administrator.'), field('Escalation policy ID', sbf.policy, 'Who gets paged when a monitor goes down. Leave blank to page nobody.')), sbPublic, sbPriv, + h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick: act(async () => { await save(); const r = await api('POST', '/api/admin/settings/sandbar-sync'); sbOut.textContent = 'Synced, ' + r.monitors + ' monitors.'; toast('Sandbar monitors synced'); })}, 'Save and sync now')), sbOut), h('div', {class:'actions form-actions', style:'margin-bottom:24px'}, h('button', {class:'btn', onclick:save}, 'Save settings')), card('Updates', h('p', {class:'dim'}, 'Hammerhead, Blacktip and Bamboo can update themselves from new releases when auto-update is on (HAMMERHEAD_AUTO_UPDATE=1). This checks the newest release for Hammerhead; it installs nothing.'), updOut, h('div', {class:'actions', style:'margin-top:10px'}, h('button', {class:'ghost', onclick:checkUpdate}, 'Check for updates'), updNow), updBox));