Hammerhead/sandbar.go

package hammerhead

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"

	"git.veillax.com/Veillax/reef/model"
	"git.veillax.com/Veillax/reef/netguard"
)

// Sandbar monitors: with this switched on under Server, Settings, every enabled
// resource that has health checks gets a monitor on a Sandbar server, which then
// tracks its uptime, shows it on the status page and pages whoever is on call.
// Sandbar probes the public address (so it sees what visitors see), where
// Hammerhead's own health checks look at the targets behind it. Hammerhead only
// ever changes the monitors it created: the link from resource to monitor is
// kept in the settings, so a monitor you add in Sandbar by hand is never touched.

// SandbarSync is the connection to Sandbar for creating monitors.
type SandbarSync struct {
	Enabled      bool              `json:"enabled"`
	URL          string            `json:"url"`
	Token        string            `json:"token"`              // an administrator's API key (secret, never sent back to the browser)
	PolicyID     string            `json:"policyId"`           // escalation policy for the alerts the monitors raise; blank pages nobody
	Public       bool              `json:"public"`             // show the monitors on Sandbar's status page
	AllowPrivate bool              `json:"allowPrivate"`       // Sandbar is on a private network, and so are the edge addresses it will probe
	Monitors     map[string]string `json:"monitors,omitempty"` // resource id -> Sandbar monitor id
}

// sandbarSyncStatus is how the last sync went, for the settings page.
type sandbarSyncStatus struct {
	At       time.Time `json:"at"`
	Error    string    `json:"error"`
	Monitors int       `json:"monitors"`
}

const sandbarSyncEvery = time.Minute

type monitorSpec struct {
	Name      string
	Kind      string
	Target    string
	IntervalS int
	Expect    string
}

// sandbarEdgeHost is the address visitors reach a resource's edge on: the address it is
// bound to, else the first public address the edge reported, else the host of its
// tunnel endpoint.
func sandbarEdgeHost(e *model.Edge, bind string) string {
	if ip := net.ParseIP(bind); ip != nil && !ip.IsUnspecified() {
		return bind
	}
	var first string
	for _, a := range e.IPs {
		ip := net.ParseIP(a)
		if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
			continue
		}
		if !ip.IsPrivate() {
			return a
		}
		if first == "" {
			first = a
		}
	}
	if first != "" {
		return first
	}
	host, _, _ := net.SplitHostPort(e.TunnelAddr)
	return host
}

// monitorFor describes the Sandbar monitor a resource should have, or false when
// it has nothing worth probing (a redirect, UDP, health checks off, a wildcard).
func (s *Server) monitorFor(m *model.Mapping) (monitorSpec, bool) {
	if !m.Enabled || m.HealthSeconds == -1 || m.RedirectTo != "" || m.Proto == model.ProtoUDP {
		return monitorSpec{}, false
	}
	e := s.findEdge(m.EdgeID)
	if e == nil {
		return monitorSpec{}, false
	}
	spec := monitorSpec{Name: m.Name, IntervalS: 30}
	if m.HealthSeconds > 0 {
		spec.IntervalS = max(m.HealthSeconds, 10)
	}
	host := m.Hostname
	if strings.HasPrefix(host, "*") {
		return monitorSpec{}, false
	}
	if host == "" {
		if host = sandbarEdgeHost(e, m.BindIP); host == "" {
			return monitorSpec{}, false
		}
	}
	addr := net.JoinHostPort(host, strconv.Itoa(m.Port))
	switch m.Proto {
	case model.ProtoHTTP, model.ProtoHTTPS:
		u := url.URL{Scheme: m.Proto, Host: addr, Path: m.HealthPath}
		if (m.Proto == model.ProtoHTTP && m.Port == 80) || (m.Proto == model.ProtoHTTPS && m.Port == 443) {
			u.Host = strings.Trim(host, "[]") // a bare IPv6 address needs its brackets back
			if strings.Contains(host, ":") {
				u.Host = "[" + u.Host + "]"
			}
		}
		spec.Kind, spec.Target = "http", u.String()
		if m.Auth.Protected() {
			spec.Expect = "200-399" // visitors are sent to a sign-in page, which is the site working
		}
	default:
		spec.Kind, spec.Target = "tcp", addr
	}
	return spec, true
}

// desiredMonitorsLocked is the monitor every qualifying resource should have, by resource id.
func (s *Server) desiredMonitorsLocked() map[string]monitorSpec {
	out := map[string]monitorSpec{}
	for _, m := range s.st.Mappings {
		if spec, ok := s.monitorFor(m); ok {
			out[m.ID] = spec
		}
	}
	return out
}

func sandbarCall(cfg SandbarSync, method, path string, body any, out any) error {
	var rd io.Reader
	if body != nil {
		raw, _ := json.Marshal(body)
		rd = bytes.NewReader(raw)
	}
	req, err := http.NewRequest(method, strings.TrimRight(cfg.URL, "/")+path, rd)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", "Bearer "+cfg.Token)
	resp, err := netguard.Client(cfg.AllowPrivate).Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	switch {
	case resp.StatusCode == 401:
		return errors.New("Sandbar rejected the API key")
	case resp.StatusCode == 403:
		return errors.New("Sandbar needs an administrator's API key to manage monitors")
	case resp.StatusCode == 404 && method == "DELETE":
		return nil // already gone
	case resp.StatusCode >= 300:
		msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
		return fmt.Errorf("Sandbar answered %s %s", resp.Status, strings.TrimSpace(string(msg)))
	}
	if out != nil {
		return json.NewDecoder(resp.Body).Decode(out)
	}
	return nil
}

// syncSandbar makes Sandbar's monitors match the resources: it creates the ones
// that are missing, corrects the ones that drifted, and removes those whose
// resource is gone or no longer has health checks.
func (s *Server) syncSandbar() (err error) {
	s.sandbarMu.Lock()
	defer s.sandbarMu.Unlock()
	s.mu.Lock()
	cfg := s.st.Settings.Sandbar
	if !cfg.Enabled {
		s.mu.Unlock()
		return nil
	}
	want := s.desiredMonitorsLocked()
	s.mu.Unlock()
	links := map[string]string{}
	for k, v := range cfg.Monitors {
		links[k] = v
	}
	defer func() {
		st := sandbarSyncStatus{At: time.Now(), Monitors: len(links)}
		if err != nil {
			st.Error = err.Error()
		}
		s.mu.Lock()
		s.sandbarStatus = st
		if now := &s.st.Settings.Sandbar; now.URL == cfg.URL && !sameLinks(now.Monitors, links) {
			now.Monitors = links // kept even after a failure: a monitor already created must not be created twice
			s.save()
		}
		s.mu.Unlock()
	}()

	var remote []map[string]any
	if err := sandbarCall(cfg, "GET", "/api/v1/monitors", nil, &remote); err != nil {
		return err
	}
	byID := map[string]map[string]any{}
	for _, m := range remote {
		if id, _ := m["id"].(string); id != "" {
			byID[id] = m
		}
	}

	ids := make([]string, 0, len(want))
	for id := range want {
		ids = append(ids, id)
	}
	sort.Strings(ids)
	var errs []error
	for _, id := range ids {
		spec := want[id]
		fields := map[string]any{"name": spec.Name, "kind": spec.Kind, "target": spec.Target, "intervalS": spec.IntervalS, "expect": spec.Expect,
			"policyId": cfg.PolicyID, "public": cfg.Public, "allowPrivate": cfg.AllowPrivate, "enabled": true}
		cur, exists := byID[links[id]]
		if !exists {
			var made struct{ ID string }
			if err := sandbarCall(cfg, "POST", "/api/v1/monitors", fields, &made); err != nil {
				errs = append(errs, fmt.Errorf("%s: %w", spec.Name, err))
				continue
			}
			links[id] = made.ID
			continue
		}
		drift := false
		for k, v := range fields {
			if !sameJSON(cur[k], v) {
				cur[k], drift = v, true
			}
		}
		if drift { // the whole document is replaced, so what Sandbar added (token, retries) rides along
			if err := sandbarCall(cfg, "PUT", "/api/v1/monitors/"+links[id], cur, nil); err != nil {
				errs = append(errs, fmt.Errorf("%s: %w", spec.Name, err))
			}
		}
	}
	for id, rid := range links {
		if _, still := want[id]; still {
			continue
		}
		if err := sandbarCall(cfg, "DELETE", "/api/v1/monitors/"+rid, nil, nil); err != nil {
			errs = append(errs, err)
			continue
		}
		delete(links, id)
	}
	return errors.Join(errs...)
}

func sameJSON(a, b any) bool {
	x, _ := json.Marshal(a)
	y, _ := json.Marshal(b)
	if string(x) == "null" { // a field Sandbar left out is its zero value
		x = []byte(`""`)
		if _, isNum := b.(int); isNum {
			x = []byte("0")
		} else if _, isBool := b.(bool); isBool {
			x = []byte("false")
		}
	}
	return bytes.Equal(x, y)
}

func sameLinks(a, b map[string]string) bool {
	if len(a) != len(b) {
		return false
	}
	for k, v := range a {
		if b[k] != v {
			return false
		}
	}
	return true
}

func (s *Server) sandbarLoop() {
	t := time.NewTicker(sandbarSyncEvery)
	defer t.Stop()
	for {
		select {
		case <-s.stop:
			return
		case <-t.C:
			if err := s.syncSandbar(); err != nil {
				log.Printf("sandbar monitors: %v", err)
			}
		}
	}
}

// syncSandbarNow is the "Sync now" button: the same sync, with its outcome shown.
func (s *Server) syncSandbarNow(w http.ResponseWriter, r *http.Request, p *principal) {
	s.mu.Lock()
	on := s.st.Settings.Sandbar.Enabled
	s.mu.Unlock()
	if !on {
		fail(w, 400, errors.New("Turn on Sandbar monitors and save the settings first."))
		return
	}
	if err := s.syncSandbar(); err != nil {
		fail(w, 400, err)
		return
	}
	s.mu.Lock()
	st := s.sandbarStatus
	s.mu.Unlock()
	writeJSON(w, 200, st)
}

// cleanSandbar validates the settings in `in`, filling in what a blank field means.
// Caller holds s.mu.
func (s *Server) cleanSandbar(in *SandbarSync) error {
	old := s.st.Settings.Sandbar
	in.URL, in.PolicyID = strings.TrimRight(strings.TrimSpace(in.URL), "/"), strings.TrimSpace(in.PolicyID)
	if in.Token = strings.TrimSpace(in.Token); in.Token == "" {
		in.Token = old.Token // blank keeps the saved API key
	}
	in.Monitors = old.Monitors // not editable here
	if in.URL != old.URL {
		in.Monitors = nil // those belong to the previous server
	}
	if !in.Enabled {
		return nil
	}
	if u, err := url.Parse(in.URL); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
		return errors.New("enter Sandbar's address, such as https://sandbar.example.com")
	}
	if in.Token == "" {
		return errors.New("enter an administrator's API key from Sandbar's Account page")
	}
	return nil
}