Publish alerts to a Sandbar topic and keep a Sandbar monitor for every resource with health checks
A Sandbar topic channel publishes alerts as ntfy-style messages (API key optional). Under Server, Settings, Hammerhead can create, update and remove a Sandbar monitor per enabled resource with health checks, linked by resource id so monitors made by hand are left alone. The key is sealed at rest. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
VeillaxandClaude Sonnet 5.5 committed
1 parent
39659f0144
commit
654cf6d0fa
10 files changed
+810
-25
No files matched your search
+343
@@ -0,0 +1,343 @@
|
||||
package hammerhead
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.veillax.com/Veillax/reef/model"
|
||||
"git.veillax.com/Veillax/reef/netguard"
|
||||
)
|
||||
|
||||
// Sandbar monitors: with this switched on under Server, Settings, every enabled
|
||||
// resource that has health checks gets a monitor on a Sandbar server, which then
|
||||
// tracks its uptime, shows it on the status page and pages whoever is on call.
|
||||
// Sandbar probes the public address (so it sees what visitors see), where
|
||||
// Hammerhead's own health checks look at the targets behind it. Hammerhead only
|
||||
// ever changes the monitors it created: the link from resource to monitor is
|
||||
// kept in the settings, so a monitor you add in Sandbar by hand is never touched.
|
||||
|
||||
// SandbarSync is the connection to Sandbar for creating monitors.
|
||||
type SandbarSync struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
URL string `json:"url"`
|
||||
Token string `json:"token"` // an administrator's API key (secret, never sent back to the browser)
|
||||
PolicyID string `json:"policyId"` // escalation policy for the alerts the monitors raise; blank pages nobody
|
||||
Public bool `json:"public"` // show the monitors on Sandbar's status page
|
||||
AllowPrivate bool `json:"allowPrivate"` // Sandbar is on a private network, and so are the edge addresses it will probe
|
||||
Monitors map[string]string `json:"monitors,omitempty"` // resource id -> Sandbar monitor id
|
||||
}
|
||||
|
||||
// sandbarSyncStatus is how the last sync went, for the settings page.
|
||||
type sandbarSyncStatus struct {
|
||||
At time.Time `json:"at"`
|
||||
Error string `json:"error"`
|
||||
Monitors int `json:"monitors"`
|
||||
}
|
||||
|
||||
const sandbarSyncEvery = time.Minute
|
||||
|
||||
type monitorSpec struct {
|
||||
Name string
|
||||
Kind string
|
||||
Target string
|
||||
IntervalS int
|
||||
Expect string
|
||||
}
|
||||
|
||||
// edgeHost is the address visitors reach a resource's edge on: the address it is
|
||||
// bound to, else the first public address the edge reported, else the host of its
|
||||
// tunnel endpoint.
|
||||
func edgeHost(e *model.Edge, bind string) string {
|
||||
if ip := net.ParseIP(bind); ip != nil && !ip.IsUnspecified() {
|
||||
return bind
|
||||
}
|
||||
var first string
|
||||
for _, a := range e.IPs {
|
||||
ip := net.ParseIP(a)
|
||||
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
if !ip.IsPrivate() {
|
||||
return a
|
||||
}
|
||||
if first == "" {
|
||||
first = a
|
||||
}
|
||||
}
|
||||
if first != "" {
|
||||
return first
|
||||
}
|
||||
host, _, _ := net.SplitHostPort(e.TunnelAddr)
|
||||
return host
|
||||
}
|
||||
|
||||
// monitorFor describes the Sandbar monitor a resource should have, or false when
|
||||
// it has nothing worth probing (a redirect, UDP, health checks off, a wildcard).
|
||||
func (s *Server) monitorFor(m *model.Mapping) (monitorSpec, bool) {
|
||||
if !m.Enabled || m.HealthSeconds == -1 || m.RedirectTo != "" || m.Proto == model.ProtoUDP {
|
||||
return monitorSpec{}, false
|
||||
}
|
||||
e := s.findEdge(m.EdgeID)
|
||||
if e == nil {
|
||||
return monitorSpec{}, false
|
||||
}
|
||||
spec := monitorSpec{Name: m.Name, IntervalS: 30}
|
||||
if m.HealthSeconds > 0 {
|
||||
spec.IntervalS = max(m.HealthSeconds, 10)
|
||||
}
|
||||
host := m.Hostname
|
||||
if strings.HasPrefix(host, "*") {
|
||||
return monitorSpec{}, false
|
||||
}
|
||||
if host == "" {
|
||||
if host = edgeHost(e, m.BindIP); host == "" {
|
||||
return monitorSpec{}, false
|
||||
}
|
||||
}
|
||||
addr := net.JoinHostPort(host, strconv.Itoa(m.Port))
|
||||
switch m.Proto {
|
||||
case model.ProtoHTTP, model.ProtoHTTPS:
|
||||
u := url.URL{Scheme: m.Proto, Host: addr, Path: m.HealthPath}
|
||||
if (m.Proto == model.ProtoHTTP && m.Port == 80) || (m.Proto == model.ProtoHTTPS && m.Port == 443) {
|
||||
u.Host = strings.Trim(host, "[]") // a bare IPv6 address needs its brackets back
|
||||
if strings.Contains(host, ":") {
|
||||
u.Host = "[" + u.Host + "]"
|
||||
}
|
||||
}
|
||||
spec.Kind, spec.Target = "http", u.String()
|
||||
if m.Auth.Protected() {
|
||||
spec.Expect = "200-399" // visitors are sent to a sign-in page, which is the site working
|
||||
}
|
||||
default:
|
||||
spec.Kind, spec.Target = "tcp", addr
|
||||
}
|
||||
return spec, true
|
||||
}
|
||||
|
||||
// desiredMonitorsLocked is the monitor every qualifying resource should have, by resource id.
|
||||
func (s *Server) desiredMonitorsLocked() map[string]monitorSpec {
|
||||
out := map[string]monitorSpec{}
|
||||
for _, m := range s.st.Mappings {
|
||||
if spec, ok := s.monitorFor(m); ok {
|
||||
out[m.ID] = spec
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sandbarCall(cfg SandbarSync, method, path string, body any, out any) error {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
raw, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(raw)
|
||||
}
|
||||
req, err := http.NewRequest(method, strings.TrimRight(cfg.URL, "/")+path, rd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+cfg.Token)
|
||||
resp, err := netguard.Client(cfg.AllowPrivate).Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
switch {
|
||||
case resp.StatusCode == 401:
|
||||
return errors.New("Sandbar rejected the API key")
|
||||
case resp.StatusCode == 403:
|
||||
return errors.New("Sandbar needs an administrator's API key to manage monitors")
|
||||
case resp.StatusCode == 404 && method == "DELETE":
|
||||
return nil // already gone
|
||||
case resp.StatusCode >= 300:
|
||||
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
|
||||
return fmt.Errorf("Sandbar answered %s %s", resp.Status, strings.TrimSpace(string(msg)))
|
||||
}
|
||||
if out != nil {
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncSandbar makes Sandbar's monitors match the resources: it creates the ones
|
||||
// that are missing, corrects the ones that drifted, and removes those whose
|
||||
// resource is gone or no longer has health checks.
|
||||
func (s *Server) syncSandbar() (err error) {
|
||||
s.sandbarMu.Lock()
|
||||
defer s.sandbarMu.Unlock()
|
||||
s.mu.Lock()
|
||||
cfg := s.st.Settings.Sandbar
|
||||
if !cfg.Enabled {
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
want := s.desiredMonitorsLocked()
|
||||
s.mu.Unlock()
|
||||
links := map[string]string{}
|
||||
for k, v := range cfg.Monitors {
|
||||
links[k] = v
|
||||
}
|
||||
defer func() {
|
||||
st := sandbarSyncStatus{At: time.Now(), Monitors: len(links)}
|
||||
if err != nil {
|
||||
st.Error = err.Error()
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.sandbarStatus = st
|
||||
if now := &s.st.Settings.Sandbar; now.URL == cfg.URL && !sameLinks(now.Monitors, links) {
|
||||
now.Monitors = links // kept even after a failure: a monitor already created must not be created twice
|
||||
s.save()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
var remote []map[string]any
|
||||
if err := sandbarCall(cfg, "GET", "/api/v1/monitors", nil, &remote); err != nil {
|
||||
return err
|
||||
}
|
||||
byID := map[string]map[string]any{}
|
||||
for _, m := range remote {
|
||||
if id, _ := m["id"].(string); id != "" {
|
||||
byID[id] = m
|
||||
}
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(want))
|
||||
for id := range want {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
var errs []error
|
||||
for _, id := range ids {
|
||||
spec := want[id]
|
||||
fields := map[string]any{"name": spec.Name, "kind": spec.Kind, "target": spec.Target, "intervalS": spec.IntervalS, "expect": spec.Expect,
|
||||
"policyId": cfg.PolicyID, "public": cfg.Public, "allowPrivate": cfg.AllowPrivate, "enabled": true}
|
||||
cur, exists := byID[links[id]]
|
||||
if !exists {
|
||||
var made struct{ ID string }
|
||||
if err := sandbarCall(cfg, "POST", "/api/v1/monitors", fields, &made); err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: %w", spec.Name, err))
|
||||
continue
|
||||
}
|
||||
links[id] = made.ID
|
||||
continue
|
||||
}
|
||||
drift := false
|
||||
for k, v := range fields {
|
||||
if !sameJSON(cur[k], v) {
|
||||
cur[k], drift = v, true
|
||||
}
|
||||
}
|
||||
if drift { // the whole document is replaced, so what Sandbar added (token, retries) rides along
|
||||
if err := sandbarCall(cfg, "PUT", "/api/v1/monitors/"+links[id], cur, nil); err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: %w", spec.Name, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
for id, rid := range links {
|
||||
if _, still := want[id]; still {
|
||||
continue
|
||||
}
|
||||
if err := sandbarCall(cfg, "DELETE", "/api/v1/monitors/"+rid, nil, nil); err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
delete(links, id)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func sameJSON(a, b any) bool {
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
if string(x) == "null" { // a field Sandbar left out is its zero value
|
||||
x = []byte(`""`)
|
||||
if _, isNum := b.(int); isNum {
|
||||
x = []byte("0")
|
||||
} else if _, isBool := b.(bool); isBool {
|
||||
x = []byte("false")
|
||||
}
|
||||
}
|
||||
return bytes.Equal(x, y)
|
||||
}
|
||||
|
||||
func sameLinks(a, b map[string]string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for k, v := range a {
|
||||
if b[k] != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) sandbarLoop() {
|
||||
t := time.NewTicker(sandbarSyncEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-s.stop:
|
||||
return
|
||||
case <-t.C:
|
||||
if err := s.syncSandbar(); err != nil {
|
||||
log.Printf("sandbar monitors: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// syncSandbarNow is the "Sync now" button: the same sync, with its outcome shown.
|
||||
func (s *Server) syncSandbarNow(w http.ResponseWriter, r *http.Request, p *principal) {
|
||||
s.mu.Lock()
|
||||
on := s.st.Settings.Sandbar.Enabled
|
||||
s.mu.Unlock()
|
||||
if !on {
|
||||
fail(w, 400, errors.New("Turn on Sandbar monitors and save the settings first."))
|
||||
return
|
||||
}
|
||||
if err := s.syncSandbar(); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
st := s.sandbarStatus
|
||||
s.mu.Unlock()
|
||||
writeJSON(w, 200, st)
|
||||
}
|
||||
|
||||
// cleanSandbar validates the settings in `in`, filling in what a blank field means.
|
||||
// Caller holds s.mu.
|
||||
func (s *Server) cleanSandbar(in *SandbarSync) error {
|
||||
old := s.st.Settings.Sandbar
|
||||
in.URL, in.PolicyID = strings.TrimRight(strings.TrimSpace(in.URL), "/"), strings.TrimSpace(in.PolicyID)
|
||||
if in.Token = strings.TrimSpace(in.Token); in.Token == "" {
|
||||
in.Token = old.Token // blank keeps the saved API key
|
||||
}
|
||||
in.Monitors = old.Monitors // not editable here
|
||||
if in.URL != old.URL {
|
||||
in.Monitors = nil // those belong to the previous server
|
||||
}
|
||||
if !in.Enabled {
|
||||
return nil
|
||||
}
|
||||
if u, err := url.Parse(in.URL); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return errors.New("enter Sandbar's address, such as https://sandbar.example.com")
|
||||
}
|
||||
if in.Token == "" {
|
||||
return errors.New("enter an administrator's API key from Sandbar's Account page")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user