package hammerhead import ( "bytes" "encoding/json" "errors" "fmt" "io" "log" "net" "net/http" "net/url" "sort" "strconv" "strings" "time" "git.veillax.com/Veillax/reef/model" "git.veillax.com/Veillax/reef/netguard" ) // Sandbar monitors: with this switched on under Server, Settings, every enabled // resource that has health checks gets a monitor on a Sandbar server, which then // tracks its uptime, shows it on the status page and pages whoever is on call. // Sandbar probes the public address (so it sees what visitors see), where // Hammerhead's own health checks look at the targets behind it. Hammerhead only // ever changes the monitors it created: the link from resource to monitor is // kept in the settings, so a monitor you add in Sandbar by hand is never touched. // SandbarSync is the connection to Sandbar for creating monitors. type SandbarSync struct { Enabled bool `json:"enabled"` URL string `json:"url"` Token string `json:"token"` // an administrator's API key (secret, never sent back to the browser) PolicyID string `json:"policyId"` // escalation policy for the alerts the monitors raise; blank pages nobody Public bool `json:"public"` // show the monitors on Sandbar's status page AllowPrivate bool `json:"allowPrivate"` // Sandbar is on a private network, and so are the edge addresses it will probe Monitors map[string]string `json:"monitors,omitempty"` // resource id -> Sandbar monitor id } // sandbarSyncStatus is how the last sync went, for the settings page. type sandbarSyncStatus struct { At time.Time `json:"at"` Error string `json:"error"` Monitors int `json:"monitors"` } const sandbarSyncEvery = time.Minute type monitorSpec struct { Name string Kind string Target string IntervalS int Expect string } // sandbarEdgeHost is the address visitors reach a resource's edge on: the address it is // bound to, else the first public address the edge reported, else the host of its // tunnel endpoint. func sandbarEdgeHost(e *model.Edge, bind string) string { if ip := net.ParseIP(bind); ip != nil && !ip.IsUnspecified() { return bind } var first string for _, a := range e.IPs { ip := net.ParseIP(a) if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() { continue } if !ip.IsPrivate() { return a } if first == "" { first = a } } if first != "" { return first } host, _, _ := net.SplitHostPort(e.TunnelAddr) return host } // monitorFor describes the Sandbar monitor a resource should have, or false when // it has nothing worth probing (a redirect, UDP, health checks off, a wildcard). func (s *Server) monitorFor(m *model.Mapping) (monitorSpec, bool) { if !m.Enabled || m.HealthSeconds == -1 || m.RedirectTo != "" || m.Proto == model.ProtoUDP { return monitorSpec{}, false } e := s.findEdge(m.EdgeID) if e == nil { return monitorSpec{}, false } spec := monitorSpec{Name: m.Name, IntervalS: 30} if m.HealthSeconds > 0 { spec.IntervalS = max(m.HealthSeconds, 10) } host := m.Hostname if strings.HasPrefix(host, "*") { return monitorSpec{}, false } if host == "" { if host = sandbarEdgeHost(e, m.BindIP); host == "" { return monitorSpec{}, false } } addr := net.JoinHostPort(host, strconv.Itoa(m.Port)) switch m.Proto { case model.ProtoHTTP, model.ProtoHTTPS: u := url.URL{Scheme: m.Proto, Host: addr, Path: m.HealthPath} if (m.Proto == model.ProtoHTTP && m.Port == 80) || (m.Proto == model.ProtoHTTPS && m.Port == 443) { u.Host = strings.Trim(host, "[]") // a bare IPv6 address needs its brackets back if strings.Contains(host, ":") { u.Host = "[" + u.Host + "]" } } spec.Kind, spec.Target = "http", u.String() if m.Auth.Protected() { spec.Expect = "200-399" // visitors are sent to a sign-in page, which is the site working } default: spec.Kind, spec.Target = "tcp", addr } return spec, true } // desiredMonitorsLocked is the monitor every qualifying resource should have, by resource id. func (s *Server) desiredMonitorsLocked() map[string]monitorSpec { out := map[string]monitorSpec{} for _, m := range s.st.Mappings { if spec, ok := s.monitorFor(m); ok { out[m.ID] = spec } } return out } func sandbarCall(cfg SandbarSync, method, path string, body any, out any) error { var rd io.Reader if body != nil { raw, _ := json.Marshal(body) rd = bytes.NewReader(raw) } req, err := http.NewRequest(method, strings.TrimRight(cfg.URL, "/")+path, rd) if err != nil { return err } req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+cfg.Token) resp, err := netguard.Client(cfg.AllowPrivate).Do(req) if err != nil { return err } defer resp.Body.Close() switch { case resp.StatusCode == 401: return errors.New("Sandbar rejected the API key") case resp.StatusCode == 403: return errors.New("Sandbar needs an administrator's API key to manage monitors") case resp.StatusCode == 404 && method == "DELETE": return nil // already gone case resp.StatusCode >= 300: msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300)) return fmt.Errorf("Sandbar answered %s %s", resp.Status, strings.TrimSpace(string(msg))) } if out != nil { return json.NewDecoder(resp.Body).Decode(out) } return nil } // syncSandbar makes Sandbar's monitors match the resources: it creates the ones // that are missing, corrects the ones that drifted, and removes those whose // resource is gone or no longer has health checks. func (s *Server) syncSandbar() (err error) { s.sandbarMu.Lock() defer s.sandbarMu.Unlock() s.mu.Lock() cfg := s.st.Settings.Sandbar if !cfg.Enabled { s.mu.Unlock() return nil } want := s.desiredMonitorsLocked() s.mu.Unlock() links := map[string]string{} for k, v := range cfg.Monitors { links[k] = v } defer func() { st := sandbarSyncStatus{At: time.Now(), Monitors: len(links)} if err != nil { st.Error = err.Error() } s.mu.Lock() s.sandbarStatus = st if now := &s.st.Settings.Sandbar; now.URL == cfg.URL && !sameLinks(now.Monitors, links) { now.Monitors = links // kept even after a failure: a monitor already created must not be created twice s.save() } s.mu.Unlock() }() var remote []map[string]any if err := sandbarCall(cfg, "GET", "/api/v1/monitors", nil, &remote); err != nil { return err } byID := map[string]map[string]any{} for _, m := range remote { if id, _ := m["id"].(string); id != "" { byID[id] = m } } ids := make([]string, 0, len(want)) for id := range want { ids = append(ids, id) } sort.Strings(ids) var errs []error for _, id := range ids { spec := want[id] fields := map[string]any{"name": spec.Name, "kind": spec.Kind, "target": spec.Target, "intervalS": spec.IntervalS, "expect": spec.Expect, "policyId": cfg.PolicyID, "public": cfg.Public, "allowPrivate": cfg.AllowPrivate, "enabled": true} cur, exists := byID[links[id]] if !exists { var made struct{ ID string } if err := sandbarCall(cfg, "POST", "/api/v1/monitors", fields, &made); err != nil { errs = append(errs, fmt.Errorf("%s: %w", spec.Name, err)) continue } links[id] = made.ID continue } drift := false for k, v := range fields { if !sameJSON(cur[k], v) { cur[k], drift = v, true } } if drift { // the whole document is replaced, so what Sandbar added (token, retries) rides along if err := sandbarCall(cfg, "PUT", "/api/v1/monitors/"+links[id], cur, nil); err != nil { errs = append(errs, fmt.Errorf("%s: %w", spec.Name, err)) } } } for id, rid := range links { if _, still := want[id]; still { continue } if err := sandbarCall(cfg, "DELETE", "/api/v1/monitors/"+rid, nil, nil); err != nil { errs = append(errs, err) continue } delete(links, id) } return errors.Join(errs...) } func sameJSON(a, b any) bool { x, _ := json.Marshal(a) y, _ := json.Marshal(b) if string(x) == "null" { // a field Sandbar left out is its zero value x = []byte(`""`) if _, isNum := b.(int); isNum { x = []byte("0") } else if _, isBool := b.(bool); isBool { x = []byte("false") } } return bytes.Equal(x, y) } func sameLinks(a, b map[string]string) bool { if len(a) != len(b) { return false } for k, v := range a { if b[k] != v { return false } } return true } func (s *Server) sandbarLoop() { t := time.NewTicker(sandbarSyncEvery) defer t.Stop() for { select { case <-s.stop: return case <-t.C: if err := s.syncSandbar(); err != nil { log.Printf("sandbar monitors: %v", err) } } } } // syncSandbarNow is the "Sync now" button: the same sync, with its outcome shown. func (s *Server) syncSandbarNow(w http.ResponseWriter, r *http.Request, p *principal) { s.mu.Lock() on := s.st.Settings.Sandbar.Enabled s.mu.Unlock() if !on { fail(w, 400, errors.New("Turn on Sandbar monitors and save the settings first.")) return } if err := s.syncSandbar(); err != nil { fail(w, 400, err) return } s.mu.Lock() st := s.sandbarStatus s.mu.Unlock() writeJSON(w, 200, st) } // cleanSandbar validates the settings in `in`, filling in what a blank field means. // Caller holds s.mu. func (s *Server) cleanSandbar(in *SandbarSync) error { old := s.st.Settings.Sandbar in.URL, in.PolicyID = strings.TrimRight(strings.TrimSpace(in.URL), "/"), strings.TrimSpace(in.PolicyID) if in.Token = strings.TrimSpace(in.Token); in.Token == "" { in.Token = old.Token // blank keeps the saved API key } in.Monitors = old.Monitors // not editable here if in.URL != old.URL { in.Monitors = nil // those belong to the previous server } if !in.Enabled { return nil } if u, err := url.Parse(in.URL); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return errors.New("enter Sandbar's address, such as https://sandbar.example.com") } if in.Token == "" { return errors.New("enter an administrator's API key from Sandbar's Account page") } return nil }