Add Sandbar alert channel, tags for resources and sites, use Reef's secretbox

Sandbar channels raise and resolve alerts through its API, with the API key
sealed at rest and never returned to the browser. Resources and sites can be
tagged and filtered by tag; sites can be renamed. The secretbox package now
comes from Reef v0.1.9 instead of a private copy.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

alerts.go

@@ -28,14 +28,17 @@ const (
 	defaultOffline  = 90 * time.Second
 )
 
-// AlertChannel is somewhere alerts are sent: an email address or a webhook.
+// AlertChannel is somewhere alerts are sent: an email address, a webhook, or a
+// Sandbar server (which pages whoever is on call and tracks the alert).
 type AlertChannel struct {
 	ID           string `json:"id"`
 	OrgID        string `json:"orgId"` // "" = a server-level channel (for edges)
 	Name         string `json:"name"`
-	Kind         string `json:"kind"`   // "email" or "webhook"
-	Target       string `json:"target"` // address(es), or a webhook URL (secret: URLs often carry tokens)
+	Kind         string `json:"kind"`   // "email", "webhook" or "sandbar"
+	Target       string `json:"target"` // address(es), a webhook URL, or Sandbar's address (secret: URLs often carry tokens)
 	Format       string `json:"format"` // webhook body: "json", "slack", "discord" or "ntfy"
+	Token        string `json:"token"`  // Sandbar: API key (secret, never sent back to the browser)
+	Policy       string `json:"policy"` // Sandbar: escalation policy id; blank sends the alert without paging anyone
 	AllowPrivate bool   `json:"allowPrivate"`
 	Enabled      bool   `json:"enabled"`
 }
@@ -50,6 +53,7 @@ type alertState struct {
 type problem struct{ org, kind, name, msg string }
 
 type alertNote struct {
+	Key     string    `json:"key"`    // stable per problem, so a recovery can find its alert
 	Status  string    `json:"status"` // "firing", "resolved" or "test"
 	Org     string    `json:"org"`
 	Kind    string    `json:"kind"`
@@ -251,7 +255,7 @@ func (s *Server) evaluateAlerts(now time.Time) {
 				chans = append(chans, *c)
 			}
 		}
-		out = append(out, delivery{alertNote{Status: status, Org: orgName(st.OrgID), Kind: st.Kind, Name: st.Name, Message: st.Message, Since: st.Since, Time: now}, chans})
+		out = append(out, delivery{alertNote{Key: st.Key, Status: status, Org: orgName(st.OrgID), Kind: st.Kind, Name: st.Name, Message: st.Message, Since: st.Since, Time: now}, chans})
 		s.db.addAlertEvent(now, st.OrgID, st.Key, status, st.Name, st.Message)
 	}
 	keys := make([]string, 0, len(probs))
@@ -362,6 +366,8 @@ func (s *Server) send(ch AlertChannel, n alertNote) error {
 			}
 		}
 		return errors.Join(errs...)
+	case "sandbar":
+		return s.sendSandbar(ch, n)
 	case "webhook":
 		body, ctype, hdr := webhookBody(ch.Format, n)
 		req, err := http.NewRequest("POST", ch.Target, bytes.NewReader(body))
@@ -385,6 +391,47 @@ func (s *Server) send(ch AlertChannel, n alertNote) error {
 	return errors.New("unknown channel kind")
 }
 
+// sendSandbar raises the alert on a Sandbar server, or resolves it there when the
+// problem clears. Sandbar de-duplicates by key, so a reminder updates the alert
+// that is already open instead of opening another. A test raises an info alert
+// with no policy (nobody is paged) and resolves it straight away.
+func (s *Server) sendSandbar(ch AlertChannel, n alertNote) error {
+	post := func(body map[string]string) error {
+		raw, _ := json.Marshal(body)
+		req, err := http.NewRequest("POST", strings.TrimRight(ch.Target, "/")+"/api/v1/alerts", bytes.NewReader(raw))
+		if err != nil {
+			return err
+		}
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", "Bearer "+ch.Token)
+		resp, err := netguard.Client(ch.AllowPrivate).Do(req)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		switch {
+		case resp.StatusCode == 401 || resp.StatusCode == 403:
+			return errors.New("Sandbar rejected the API key")
+		case resp.StatusCode >= 300:
+			return fmt.Errorf("Sandbar answered %s", resp.Status)
+		}
+		return nil
+	}
+	key := "hammerhead:" + n.Key
+	switch n.Status {
+	case "resolved":
+		return post(map[string]string{"key": key, "status": "resolved", "source": "hammerhead"})
+	case "test":
+		key = "hammerhead:test:" + ch.ID
+		if err := post(map[string]string{"key": key, "source": "hammerhead", "summary": "Hammerhead test alert", "detail": n.text(), "severity": "info"}); err != nil {
+			return err
+		}
+		return post(map[string]string{"key": key, "status": "resolved", "source": "hammerhead"})
+	}
+	return post(map[string]string{"key": key, "source": "hammerhead", "summary": n.Name + ": " + n.Message,
+		"detail": "Organization: " + n.Org + "\nSince: " + n.Since.Format(time.RFC1123), "severity": "critical", "policy": ch.Policy})
+}
+
 func webhookBody(format string, n alertNote) (body []byte, ctype string, hdr map[string]string) {
 	text := n.subject() + "\n" + n.text()
 	switch format {
@@ -415,12 +462,25 @@ func (s *Server) validateChannel(ch *AlertChannel, trusted bool) error {
 				return errors.New("enter one or more email addresses, separated by commas")
 			}
 		}
-		ch.Format, ch.AllowPrivate = "", false
+		ch.Format, ch.Token, ch.Policy, ch.AllowPrivate = "", "", "", false
+	case "sandbar":
+		u, err := url.Parse(ch.Target)
+		if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
+			return errors.New("enter Sandbar's address, such as https://sandbar.example.com")
+		}
+		if ch.Token = strings.TrimSpace(ch.Token); ch.Token == "" {
+			return errors.New("enter an API key from Sandbar's Account page")
+		}
+		ch.Policy, ch.Format = strings.TrimSpace(ch.Policy), ""
+		if ch.AllowPrivate && !trusted {
+			return errors.New("sending to private addresses (like a service on your home network) needs a server administrator to allow it for this organization")
+		}
 	case "webhook":
 		u, err := url.Parse(ch.Target)
 		if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
 			return errors.New("the webhook must be an http:// or https:// URL")
 		}
+		ch.Token, ch.Policy = "", ""
 		switch ch.Format {
 		case "json", "slack", "discord", "ntfy":
 		case "":
@@ -432,7 +492,7 @@ func (s *Server) validateChannel(ch *AlertChannel, trusted bool) error {
 			return errors.New("sending to private addresses (like a service on your home network) needs a server administrator to allow it for this organization")
 		}
 	default:
-		return errors.New("a channel is an email or a webhook")
+		return errors.New("a channel is an email, a webhook or a Sandbar server")
 	}
 	if ch.Name == "" {
 		ch.Name = ch.Kind