A Sandbar topic channel publishes alerts as ntfy-style messages (API key optional). Under Server, Settings, Hammerhead can create, update and remove a Sandbar monitor per enabled resource with health checks, linked by resource id so monitors made by hand are left alone. The key is sealed at rest. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
351 lines
11 KiB
Go
351 lines
11 KiB
Go
package hammerhead
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.veillax.com/Veillax/reef/model"
|
|
)
|
|
|
|
// fakeSandbar keeps monitors in memory the way Sandbar's /api/v1/monitors does.
|
|
type fakeSandbar struct {
|
|
URL string
|
|
mu sync.Mutex
|
|
mon map[string]map[string]any
|
|
n int
|
|
// requests that carried the wrong key are counted, not served
|
|
badAuth int
|
|
calls []string
|
|
}
|
|
|
|
func newFakeSandbar(t *testing.T) *fakeSandbar {
|
|
f := &fakeSandbar{mon: map[string]map[string]any{}}
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if r.Header.Get("Authorization") != "Bearer sb_admin" {
|
|
f.badAuth++
|
|
http.Error(w, "no", 401)
|
|
return
|
|
}
|
|
f.calls = append(f.calls, r.Method)
|
|
id := strings.TrimPrefix(r.URL.Path, "/api/v1/monitors/")
|
|
switch {
|
|
case r.Method == "GET" && r.URL.Path == "/api/v1/monitors":
|
|
out := []any{}
|
|
for _, m := range f.mon {
|
|
out = append(out, m)
|
|
}
|
|
json.NewEncoder(w).Encode(out)
|
|
case r.Method == "POST" && r.URL.Path == "/api/v1/monitors":
|
|
var m map[string]any
|
|
json.NewDecoder(r.Body).Decode(&m)
|
|
f.n++
|
|
m["id"], m["retries"] = "mon"+strconv.Itoa(f.n), 2 // Sandbar adds its own fields
|
|
f.mon[m["id"].(string)] = m
|
|
json.NewEncoder(w).Encode(m)
|
|
case r.Method == "PUT" && f.mon[id] != nil:
|
|
var m map[string]any
|
|
json.NewDecoder(r.Body).Decode(&m)
|
|
f.mon[id] = m
|
|
json.NewEncoder(w).Encode(m)
|
|
case r.Method == "DELETE":
|
|
delete(f.mon, id)
|
|
w.Write([]byte(`{"ok":true}`))
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
t.Cleanup(ts.Close)
|
|
f.URL = ts.URL
|
|
return f
|
|
}
|
|
|
|
func (f *fakeSandbar) names() map[string]string { // monitor id -> "kind target"
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
out := map[string]string{}
|
|
for id, m := range f.mon {
|
|
out[id] = m["kind"].(string) + " " + m["target"].(string)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (f *fakeSandbar) count(method string) int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
n := 0
|
|
for _, c := range f.calls {
|
|
if c == method {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func enableSandbarSync(f *fx, sb *fakeSandbar) {
|
|
f.s.mu.Lock()
|
|
f.s.st.Edges[0].IPs = []string{"10.0.0.5", "203.0.113.7"}
|
|
f.s.st.Settings.Sandbar = SandbarSync{Enabled: true, URL: sb.URL, Token: "sb_admin", PolicyID: "pol1", Public: true, AllowPrivate: true}
|
|
f.s.mu.Unlock()
|
|
}
|
|
|
|
func TestSandbarMonitorsFollowTheResources(t *testing.T) {
|
|
f := newFx(t)
|
|
sb := newFakeSandbar(t)
|
|
enableSandbarSync(f, sb)
|
|
f.s.mu.Lock()
|
|
f.s.st.Mappings = append(f.s.st.Mappings,
|
|
&model.Mapping{ID: "m2", OrgID: f.org, Name: "Blog", EdgeID: "e1", Proto: "https", Port: 443, Hostname: "blog.example.com", HealthPath: "/ping", Enabled: true,
|
|
Targets: []model.Target{{SiteID: "s1", Host: "192.168.1.31", Port: 80}}},
|
|
&model.Mapping{ID: "m3", OrgID: f.org, Name: "Docs", EdgeID: "e1", Proto: "https", Port: 8443, Hostname: "docs.example.com", Enabled: true, Auth: model.Auth{Password: true},
|
|
Targets: []model.Target{{SiteID: "s1", Host: "192.168.1.32", Port: 80}}},
|
|
&model.Mapping{ID: "m4", OrgID: f.org, Name: "Unchecked", EdgeID: "e1", Proto: "tcp", Port: 2222, HealthSeconds: -1, Enabled: true},
|
|
&model.Mapping{ID: "m5", OrgID: f.org, Name: "Voice", EdgeID: "e1", Proto: "udp", Port: 9987, Enabled: true},
|
|
&model.Mapping{ID: "m6", OrgID: f.org, Name: "Wildcard", EdgeID: "e1", Proto: "https", Port: 443, Hostname: "*.example.com", Enabled: true},
|
|
&model.Mapping{ID: "m7", OrgID: f.org, Name: "Off", EdgeID: "e1", Proto: "tcp", Port: 3333},
|
|
)
|
|
f.s.mu.Unlock()
|
|
|
|
if err := f.s.syncSandbar(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := sb.names()
|
|
want := map[string]bool{"tcp 203.0.113.7:25565": true, "http https://blog.example.com/ping": true, "http https://docs.example.com:8443": true}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("monitors created: %v", got)
|
|
}
|
|
for _, v := range got {
|
|
if !want[v] {
|
|
t.Errorf("unexpected monitor %q", v)
|
|
}
|
|
}
|
|
for _, m := range sb.mon {
|
|
if m["policyId"] != "pol1" || m["public"] != true || m["enabled"] != true {
|
|
t.Errorf("monitor settings not applied: %v", m)
|
|
}
|
|
if m["name"] == "Docs" && m["expect"] != "200-399" {
|
|
t.Errorf("a protected site should accept its sign-in redirect: %v", m)
|
|
}
|
|
}
|
|
|
|
// A second pass changes nothing.
|
|
posts := sb.count("POST")
|
|
if err := f.s.syncSandbar(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if sb.count("POST") != posts || sb.count("PUT") != 0 || sb.count("DELETE") != 0 {
|
|
t.Fatalf("a settled sync still wrote: %v", sb.calls)
|
|
}
|
|
|
|
// A rename and a changed policy are pushed, and Sandbar's own fields survive.
|
|
f.s.mu.Lock()
|
|
f.s.findMapping("m1").Name = "Minecraft Java"
|
|
f.s.st.Settings.Sandbar.PolicyID = "pol2"
|
|
f.s.mu.Unlock()
|
|
if err := f.s.syncSandbar(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var renamed bool
|
|
for _, m := range sb.mon {
|
|
if m["policyId"] != "pol2" || m["retries"] != float64(2) {
|
|
t.Errorf("update lost data or missed the policy: %v", m)
|
|
}
|
|
renamed = renamed || m["name"] == "Minecraft Java"
|
|
}
|
|
if !renamed {
|
|
t.Error("the rename was not pushed")
|
|
}
|
|
|
|
// Deleting a resource, or turning its checks off, removes its monitor; one added by hand stays.
|
|
sb.mu.Lock()
|
|
sb.mon["hand"] = map[string]any{"id": "hand", "name": "mine", "kind": "tcp", "target": "x:1"}
|
|
sb.mu.Unlock()
|
|
f.s.mu.Lock()
|
|
f.s.findMapping("m2").HealthSeconds = -1
|
|
f.s.st.Mappings = filter(f.s.st.Mappings, func(m *model.Mapping) bool { return m.ID != "m3" })
|
|
f.s.mu.Unlock()
|
|
if err := f.s.syncSandbar(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got = sb.names()
|
|
if len(got) != 2 || got["hand"] == "" {
|
|
t.Fatalf("after removing resources: %v", got)
|
|
}
|
|
f.s.mu.Lock()
|
|
links := len(f.s.st.Settings.Sandbar.Monitors)
|
|
f.s.mu.Unlock()
|
|
if links != 1 {
|
|
t.Errorf("links kept for %d resources, want 1", links)
|
|
}
|
|
}
|
|
|
|
func TestSandbarSyncRecreatesAMonitorDeletedInSandbarAndSurvivesARestart(t *testing.T) {
|
|
f := newFx(t)
|
|
sb := newFakeSandbar(t)
|
|
enableSandbarSync(f, sb)
|
|
if err := f.s.syncSandbar(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sb.mu.Lock()
|
|
sb.mon = map[string]map[string]any{}
|
|
sb.mu.Unlock()
|
|
if err := f.s.syncSandbar(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(sb.names()) != 1 {
|
|
t.Fatalf("not recreated: %v", sb.names())
|
|
}
|
|
|
|
// the links are stored, so a restart does not create a second monitor
|
|
f.s.mu.Lock()
|
|
err := f.s.save()
|
|
f.s.mu.Unlock()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dir, key := f.s.dir, f.s.opts.MasterKey
|
|
f.s.Close()
|
|
s2, err := New(dir, Options{MasterKey: key, AlertInterval: time.Hour})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s2.Close()
|
|
if err := s2.syncSandbar(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(sb.names()) != 1 {
|
|
t.Fatalf("a restart duplicated the monitor: %v", sb.names())
|
|
}
|
|
}
|
|
|
|
func TestSandbarSyncReportsABadKeyAndIsOffByDefault(t *testing.T) {
|
|
f := newFx(t)
|
|
sb := newFakeSandbar(t)
|
|
if err := f.s.syncSandbar(); err != nil || sb.badAuth != 0 || len(sb.calls) != 0 {
|
|
t.Fatal("sync ran while switched off")
|
|
}
|
|
enableSandbarSync(f, sb)
|
|
f.s.mu.Lock()
|
|
f.s.st.Settings.Sandbar.Token = "wrong"
|
|
f.s.mu.Unlock()
|
|
if err := f.s.syncSandbar(); err == nil || !strings.Contains(err.Error(), "API key") {
|
|
t.Fatalf("error: %v", err)
|
|
}
|
|
f.s.mu.Lock()
|
|
msg := f.s.sandbarStatus.Error
|
|
f.s.mu.Unlock()
|
|
if msg == "" {
|
|
t.Error("the failure was not recorded for the settings page")
|
|
}
|
|
}
|
|
|
|
func TestSandbarSettingsKeepTheKeySealedAndHidden(t *testing.T) {
|
|
f := newFx(t)
|
|
f.s.mu.Lock()
|
|
in := SandbarSync{Enabled: true, URL: " https://sandbar.example.com/ ", Token: " sb_admin "}
|
|
err := f.s.cleanSandbar(&in)
|
|
f.s.st.Settings.Sandbar = in
|
|
f.s.save()
|
|
f.s.mu.Unlock()
|
|
if err != nil || in.URL != "https://sandbar.example.com" || in.Token != "sb_admin" {
|
|
t.Fatalf("%v %+v", err, in)
|
|
}
|
|
|
|
// a blank key in a later save keeps the saved one; a new address forgets the old server's links
|
|
f.s.mu.Lock()
|
|
f.s.st.Settings.Sandbar.Monitors = map[string]string{"m1": "mon1"}
|
|
next := SandbarSync{Enabled: true, URL: "https://other.example.com"}
|
|
err = f.s.cleanSandbar(&next)
|
|
f.s.mu.Unlock()
|
|
if err != nil || next.Token != "sb_admin" || next.Monitors != nil {
|
|
t.Fatalf("%v %+v", err, next)
|
|
}
|
|
for _, bad := range []SandbarSync{{Enabled: true, URL: "not a url", Token: "k"}, {Enabled: true, URL: "https://x.example.com", Token: ""}} {
|
|
bad := bad
|
|
f.s.mu.Lock()
|
|
f.s.st.Settings.Sandbar.Token = ""
|
|
err := f.s.cleanSandbar(&bad)
|
|
f.s.mu.Unlock()
|
|
if err == nil {
|
|
t.Errorf("accepted %+v", bad)
|
|
}
|
|
}
|
|
|
|
// at rest the key is encrypted, and the settings API never returns it
|
|
raw, _ := json.Marshal(f.s.db.sealed(f.s.st.Settings))
|
|
if strings.Contains(string(raw), "sb_admin") {
|
|
t.Error("the API key is stored in the clear")
|
|
}
|
|
f.s.mu.Lock()
|
|
f.s.st.Settings.Sandbar.Token = "sb_admin"
|
|
f.s.mu.Unlock()
|
|
rec := httptest.NewRecorder()
|
|
f.s.getSettings(rec, httptest.NewRequest("GET", "/api/admin/settings", nil), nil)
|
|
if strings.Contains(rec.Body.String(), "sb_admin") || !strings.Contains(rec.Body.String(), `"sandbarTokenSet":true`) {
|
|
t.Errorf("settings response: %s", rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestTopicChannelPublishes(t *testing.T) {
|
|
f := newFx(t)
|
|
var gotPath, gotTitle, gotPrio, gotAuth, gotBody string
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
b := make([]byte, 512)
|
|
n, _ := r.Body.Read(b)
|
|
gotPath, gotTitle, gotPrio, gotAuth, gotBody = r.URL.Path, r.Header.Get("Title"), r.Header.Get("Priority"), r.Header.Get("Authorization"), string(b[:n])
|
|
}))
|
|
defer ts.Close()
|
|
ch := AlertChannel{ID: "t1", Kind: "topic", Target: ts.URL + "/", Topic: "ops", Token: "sb_k", AllowPrivate: true, Enabled: true}
|
|
n := alertNote{Status: "firing", Org: "Acme", Name: "Minecraft", Message: "Minecraft can't listen", Since: time.Now()}
|
|
if err := f.s.send(ch, n); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if gotPath != "/ops" || gotTitle != "[Hammerhead] Problem: Minecraft" || gotPrio != "urgent" || gotAuth != "Bearer sb_k" || !strings.Contains(gotBody, "can't listen") {
|
|
t.Errorf("published %q %q %q %q %q", gotPath, gotTitle, gotPrio, gotAuth, gotBody)
|
|
}
|
|
n.Status, ch.Token = "resolved", ""
|
|
if err := f.s.send(ch, n); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if gotPrio != "default" || gotAuth != "" || !strings.HasPrefix(gotBody, "Resolved") {
|
|
t.Errorf("recovery %q %q %q", gotPrio, gotAuth, gotBody)
|
|
}
|
|
|
|
// a refusal surfaces as an error
|
|
deny := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "no", 401) }))
|
|
defer deny.Close()
|
|
ch.Target = deny.URL
|
|
if err := f.s.send(ch, n); err == nil || !strings.Contains(err.Error(), "API key") {
|
|
t.Errorf("error: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTopicChannelValidation(t *testing.T) {
|
|
f := newFx(t)
|
|
for _, ch := range []AlertChannel{
|
|
{Kind: "topic", Target: "not a url", Topic: "ops"},
|
|
{Kind: "topic", Target: "https://sandbar.example.com", Topic: ""},
|
|
{Kind: "topic", Target: "https://sandbar.example.com", Topic: "has space"},
|
|
{Kind: "topic", Target: "https://sandbar.example.com", Topic: "ops", AllowPrivate: true}, // needs a trusted caller
|
|
} {
|
|
ch := ch
|
|
if err := f.s.validateChannel(&ch, false); err == nil {
|
|
t.Fatalf("accepted %+v", ch)
|
|
}
|
|
}
|
|
ok := AlertChannel{Kind: "topic", Target: "https://sandbar.example.com", Topic: " ops ", Token: " ", Policy: "x", Format: "slack"}
|
|
if err := f.s.validateChannel(&ok, false); err != nil || ok.Topic != "ops" || ok.Token != "" || ok.Policy != "" || ok.Format != "" {
|
|
t.Fatalf("%v %+v", err, ok)
|
|
}
|
|
if p := preview(ok); p != "https://sandbar.example.com/ops" {
|
|
t.Errorf("preview %q", p)
|
|
}
|
|
}
|