package sandbar import ( "bufio" "context" "encoding/json" "io" "net/http" "net/http/httptest" "strings" "sync" "testing" "time" "git.veillax.com/Veillax/reef/secretbox" ) type fakeClock struct { mu sync.Mutex t time.Time } func (c *fakeClock) Now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t } func (c *fakeClock) Advance(d time.Duration) { c.mu.Lock() c.t = c.t.Add(d) c.mu.Unlock() } type mailLog struct { mu sync.Mutex sent []string } func (m *mailLog) send(to, subject, body string) error { m.mu.Lock() m.sent = append(m.sent, to) m.mu.Unlock() return nil } func (m *mailLog) to() []string { m.mu.Lock() defer m.mu.Unlock() return append([]string(nil), m.sent...) } func newTestServer(t *testing.T) (*Server, *fakeClock, *mailLog) { t.Helper() box, _ := secretbox.New(secretbox.GenerateKey()) st, err := OpenStore(t.TempDir(), box) if err != nil { t.Fatal(err) } t.Cleanup(func() { st.Close() }) clk := &fakeClock{t: time.Date(2026, 1, 5, 9, 0, 0, 0, time.UTC)} ml := &mailLog{} return New(Options{Now: clk.Now, Mailer: ml.send, URL: "http://sandbar.test"}, st), clk, ml } func mustUser(t *testing.T, s *Server, name string) *User { t.Helper() u, err := s.createUser(name, strings.ToLower(name)+"@example.com", "correct-horse-battery", "member", "") if err != nil { t.Fatal(err) } return u } func TestScheduleRotationAndOverride(t *testing.T) { start := time.Date(2026, 1, 5, 0, 0, 0, 0, time.UTC) sc := &Schedule{Members: []string{"a", "b", "c"}, RotationHours: 24, Start: start.Unix()} for day, want := range []string{"a", "b", "c", "a"} { if got := sc.OnCall(start.Add(time.Duration(day)*24*time.Hour + time.Hour)); got != want { t.Errorf("day %d: on call %q, want %q", day, got, want) } } sc.Overrides = []Override{{UserID: "z", From: start.Add(2 * time.Hour).Unix(), To: start.Add(5 * time.Hour).Unix()}} if got := sc.OnCall(start.Add(3 * time.Hour)); got != "z" { t.Errorf("override: %q", got) } if got := sc.OnCall(start.Add(6 * time.Hour)); got != "a" { t.Errorf("after override: %q", got) } if (&Schedule{}).OnCall(start) != "" { t.Error("empty schedule should have nobody on call") } } func TestEscalationWalksStepsUntilAcked(t *testing.T) { s, clk, mails := newTestServer(t) alice, bob := mustUser(t, s, "Alice"), mustUser(t, s, "Bob") s.st.put("schedule", "sch", &Schedule{ID: "sch", Name: "primary", Members: []string{alice.ID}, RotationHours: 24, Start: clk.Now().Unix()}) s.st.put("policy", "pol", &Policy{ID: "pol", Name: "p", Steps: []Step{ {Schedules: []string{"sch"}}, {DelayMin: 5, Users: []string{bob.ID}}, }}) a, isNew, err := s.RaiseAlert("k1", "test", "db down", "", "critical", "pol") if err != nil || !isNew { t.Fatalf("raise: %v new=%v", err, isNew) } s.escalate(clk.Now()) if got := s.MessagesSince(userTopic(alice.ID), 0, 10); len(got) != 1 { t.Fatalf("alice should be paged once, got %d", len(got)) } if len(s.MessagesSince(userTopic(bob.ID), 0, 10)) != 0 { t.Fatal("bob must wait for step 2") } clk.Advance(4 * time.Minute) s.escalate(clk.Now()) if len(s.MessagesSince(userTopic(bob.ID), 0, 10)) != 0 { t.Fatal("step 2 fired early") } clk.Advance(2 * time.Minute) s.escalate(clk.Now()) if len(s.MessagesSince(userTopic(bob.ID), 0, 10)) != 1 { t.Fatal("bob should be paged after the delay") } // A second alert with the same key is deduplicated, not a new page. b, isNew, _ := s.RaiseAlert("k1", "test", "db down", "still", "critical", "pol") if isNew || b.ID != a.ID || b.Count != 2 { t.Fatalf("dedupe: new=%v same=%v count=%d", isNew, b.ID == a.ID, b.Count) } // Ack stops escalation entirely. c, _, _ := s.RaiseAlert("k2", "test", "disk full", "", "warning", "pol") s.escalate(clk.Now()) if err := s.AckAlert(c.ID, "Alice"); err != nil { t.Fatal(err) } clk.Advance(time.Hour) before := len(s.MessagesSince(userTopic(bob.ID), 0, 50)) s.escalate(clk.Now()) if len(s.MessagesSince(userTopic(bob.ID), 0, 50)) != before { t.Fatal("an acked alert kept escalating") } // Emails go out on a goroutine; give them a moment. deadline := time.Now().Add(2 * time.Second) for len(mails.to()) == 0 && time.Now().Before(deadline) { time.Sleep(10 * time.Millisecond) } if len(mails.to()) == 0 { t.Fatal("expected email notifications to be sent") } } func TestSnoozeResumesAtSameStep(t *testing.T) { s, clk, _ := newTestServer(t) alice := mustUser(t, s, "Alice") s.st.put("policy", "pol", &Policy{ID: "pol", Name: "p", Steps: []Step{{Users: []string{alice.ID}}}}) a, _, _ := s.RaiseAlert("k", "test", "x", "", "info", "pol") if err := s.SnoozeAlert(a.ID, "me", 30*time.Minute); err != nil { t.Fatal(err) } s.escalate(clk.Now()) if len(s.MessagesSince(userTopic(alice.ID), 0, 10)) != 0 { t.Fatal("snoozed alert paged") } clk.Advance(31 * time.Minute) s.escalate(clk.Now()) if len(s.MessagesSince(userTopic(alice.ID), 0, 10)) != 1 { t.Fatal("alert should page when the snooze ends") } } func TestMonitorHTTPDownAndRecover(t *testing.T) { s, clk, _ := newTestServer(t) var mu sync.Mutex status := 200 backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mu.Lock() defer mu.Unlock() w.WriteHeader(status) io.WriteString(w, "hello world") })) defer backend.Close() m := &Monitor{ID: "m1", Name: "web", Kind: "http", Target: backend.URL, IntervalS: 30, Retries: 2, Enabled: true, AllowPrivate: true, Keyword: "hello"} if err := validateMonitor(m); err != nil { t.Fatal(err) } s.st.put("monitor", m.ID, m) step := func() { s.checkMonitor(m, clk.Now()); clk.Advance(30 * time.Second) } setStatus := func(v int) { mu.Lock(); status = v; mu.Unlock() } step() if st := s.monitorState("m1"); st.Status != "up" { t.Fatalf("want up, got %s (%s)", st.Status, st.Detail) } setStatus(500) step() if st := s.monitorState("m1"); st.Status != "up" || st.Fails != 1 { t.Fatalf("one failure must not flip it: %+v", st) } step() if st := s.monitorState("m1"); st.Status != "down" { t.Fatalf("want down after retries, got %+v", st) } if open := s.listAlerts("open", 10); len(open) != 1 || !strings.Contains(open[0].Summary, "web is down") { t.Fatalf("expected one open alert, got %+v", open) } step() // still down: same alert, not a second one if open := s.listAlerts("open", 10); len(open) != 1 { t.Fatalf("alert duplicated: %d", len(open)) } setStatus(200) step() if st := s.monitorState("m1"); st.Status != "up" { t.Fatalf("want recovered, got %+v", st) } if open := s.listAlerts("open", 10); len(open) != 0 { t.Fatalf("alert should have resolved: %+v", open) } if up := s.monitorState("m1").Uptime30; up <= 0 || up >= 100 { t.Fatalf("uptime should reflect the outage, got %v", up) } } func TestMonitorBlocksPrivateTargetsByDefault(t *testing.T) { s, _, _ := newTestServer(t) backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) defer backend.Close() m := &Monitor{Name: "x", Kind: "http", Target: backend.URL} if _, err := s.runCheck(context.Background(), m); err == nil { t.Fatal("a loopback target must be refused unless allowPrivate is set") } } func TestHeartbeatMissed(t *testing.T) { s, clk, _ := newTestServer(t) m := &Monitor{ID: "hb", Name: "backup", Kind: "heartbeat", IntervalS: 60, GraceS: 10, Enabled: true} if err := validateMonitor(m); err != nil { t.Fatal(err) } s.st.put("monitor", m.ID, m) s.checkMonitor(m, clk.Now()) // first look: starts the clock if s.monitorState("hb").Status != "pending" { t.Fatal("a heartbeat that was never expected yet should be pending") } clk.Advance(30 * time.Second) if !s.Heartbeat(m.Token) { t.Fatal("ping rejected") } s.checkMonitor(m, clk.Now()) if s.monitorState("hb").Status != "up" { t.Fatal("want up after a ping") } clk.Advance(2 * time.Minute) for i := 0; i < 2; i++ { s.checkMonitor(m, clk.Now()) clk.Advance(15 * time.Second) } if s.monitorState("hb").Status != "down" { t.Fatalf("missed heartbeat should be down: %+v", s.monitorState("hb")) } if !s.Heartbeat(m.Token) { t.Fatal("ping rejected") } if s.monitorState("hb").Status != "up" || len(s.listAlerts("open", 10)) != 0 { t.Fatal("a ping should recover a down heartbeat immediately") } if s.Heartbeat("nope") { t.Fatal("unknown token accepted") } } func TestIncidentLifecycle(t *testing.T) { s, _, _ := newTestServer(t) u := mustUser(t, s, "Alice") a, _, _ := s.RaiseAlert("k", "t", "api 500s", "", "critical", "") in, err := s.DeclareIncident("API outage", "sev1", "errors everywhere", u.ID, u.Name, []string{a.ID}) if err != nil { t.Fatal(err) } if got, _ := s.getAlert(a.ID); got.IncidentID != in.ID { t.Fatal("alert not linked") } st := "identified" if _, err := s.UpdateIncident(in.ID, u.Name, IncidentPatch{Status: &st, Note: "bad deploy"}); err != nil { t.Fatal(err) } bogus := "exploding" if _, err := s.UpdateIncident(in.ID, u.Name, IncidentPatch{Status: &bogus}); err == nil { t.Fatal("invalid status accepted") } done := "resolved" got, _ := s.UpdateIncident(in.ID, u.Name, IncidentPatch{Status: &done}) if got.Resolved == 0 { t.Fatal("resolved time not set") } s.CommentIncident(in.ID, u.Name, "rolled back") ev := s.st.events("incident", in.ID) if len(ev) < 4 { t.Fatalf("timeline too short: %+v", ev) } if tpl := s.postmortemTemplate(got); !strings.Contains(tpl, "rolled back") || !strings.Contains(tpl, "API outage") { t.Fatalf("template missing content:\n%s", tpl) } if n := len(s.MessagesSince("incidents", 0, 10)); n != 3 { t.Fatalf("expected declare + 2 updates on the incidents topic, got %d", n) } } func TestChannelSecretsAreSealedAndRedacted(t *testing.T) { s, _, _ := newTestServer(t) ts := httptest.NewServer(s.Handler()) defer ts.Close() c := newAPIClient(t, ts) c.setup() var created map[string]any c.do("POST", "/api/v1/channels", map[string]any{"name": "slack", "kind": "webhook", "target": "https://hooks.example.com/secret-token", "format": "slack", "enabled": true}, &created) var raw string s.st.db.QueryRow(`SELECT body FROM docs WHERE kind='channel'`).Scan(&raw) if strings.Contains(raw, "secret-token") { t.Fatal("webhook URL stored in clear text") } if created["target"] == "https://hooks.example.com/secret-token" { t.Fatal("the API returned the secret") } // Editing without touching the target keeps the stored secret. created["name"] = "slack2" c.do("PUT", "/api/v1/channels/"+created["id"].(string), created, nil) ch, _ := getOf[Channel](s.st, "channel", created["id"].(string)) if s.st.open(ch.Target) != "https://hooks.example.com/secret-token" || ch.Name != "slack2" { t.Fatalf("secret lost on edit: %+v", ch) } } // ---------- HTTP API end to end ---------- type apiClient struct { t *testing.T ts *httptest.Server cookie *http.Cookie } func newAPIClient(t *testing.T, ts *httptest.Server) *apiClient { return &apiClient{t: t, ts: ts} } func (c *apiClient) req(method, path string, body any, headers map[string]string) *http.Response { c.t.Helper() var rd io.Reader if body != nil { b, _ := json.Marshal(body) rd = strings.NewReader(string(b)) } r, _ := http.NewRequest(method, c.ts.URL+path, rd) if body != nil || method != "GET" { r.Header.Set("Content-Type", "application/json") } if c.cookie != nil { r.AddCookie(c.cookie) } for k, v := range headers { r.Header.Set(k, v) } resp, err := http.DefaultClient.Do(r) if err != nil { c.t.Fatal(err) } return resp } func (c *apiClient) do(method, path string, body any, out any) int { c.t.Helper() resp := c.req(method, path, body, nil) defer resp.Body.Close() if out != nil { json.NewDecoder(resp.Body).Decode(out) } return resp.StatusCode } func (c *apiClient) setup() { c.t.Helper() resp := c.req("POST", "/api/v1/setup", map[string]string{"name": "Root", "email": "root@example.com", "password": "correct-horse-battery"}, nil) defer resp.Body.Close() if resp.StatusCode != 200 { c.t.Fatalf("setup: %d", resp.StatusCode) } for _, ck := range resp.Cookies() { if ck.Name == sessionCookie { c.cookie = ck } } if c.cookie == nil { c.t.Fatal("no session cookie") } } func TestAPIAuthSetupAndTopics(t *testing.T) { s, _, _ := newTestServer(t) ts := httptest.NewServer(s.Handler()) defer ts.Close() c := newAPIClient(t, ts) if code := c.do("GET", "/api/v1/alerts", nil, nil); code != 401 { t.Fatalf("unauthenticated API call: %d", code) } if code := c.do("POST", "/mytopic", nil, nil); code != 401 { t.Fatalf("unauthenticated publish: %d", code) } c.setup() if code := c.do("POST", "/api/v1/setup", map[string]string{"name": "x", "email": "x@x.com", "password": "correct-horse-battery"}, nil); code != 403 { t.Fatalf("second setup must be refused, got %d", code) } var key map[string]string c.do("POST", "/api/v1/apikeys", map[string]string{"name": "ci"}, &key) if !strings.HasPrefix(key["token"], "sb_") { t.Fatalf("api key: %v", key) } bearer := map[string]string{"Authorization": "Bearer " + key["token"]} // Subscribe with the API key, then publish ntfy-style with headers. sub := make(chan string, 4) go func() { req, _ := http.NewRequest("GET", ts.URL+"/alerts-room/sse", nil) req.Header.Set("Authorization", "Bearer "+key["token"]) resp, err := http.DefaultClient.Do(req) if err != nil { return } defer resp.Body.Close() sc := bufio.NewScanner(resp.Body) for sc.Scan() { if strings.HasPrefix(sc.Text(), "data: ") { sub <- sc.Text()[6:] return } } }() time.Sleep(300 * time.Millisecond) r, _ := http.NewRequest("POST", ts.URL+"/alerts-room", strings.NewReader("disk is full")) r.Header.Set("Authorization", "Bearer "+key["token"]) r.Header.Set("Title", "Disk") r.Header.Set("Priority", "urgent") r.Header.Set("Tags", "warning,disk") resp, err := http.DefaultClient.Do(r) if err != nil || resp.StatusCode != 200 { t.Fatalf("publish: %v %v", err, resp) } select { case line := <-sub: var m Message json.Unmarshal([]byte(line), &m) if m.Message != "disk is full" || m.Title != "Disk" || m.Priority != 5 || len(m.Tags) != 2 { t.Fatalf("bad message: %+v", m) } case <-time.After(3 * time.Second): t.Fatal("subscriber got nothing") } // Reserved topics are refused. for _, reserved := range []string{"/api", "/status"} { r, _ := http.NewRequest("POST", ts.URL+reserved, strings.NewReader("x")) r.Header.Set("Authorization", "Bearer "+key["token"]) resp, _ := http.DefaultClient.Do(r) if resp.StatusCode == 200 { t.Errorf("publish to %s should not succeed", reserved) } } // Inbound alert -> ack -> resolve over the API with the key. var out map[string]any c.cookie = nil if code := c.do("POST", "/api/v1/alerts", map[string]string{"key": "cpu", "summary": "CPU high"}, nil); code != 401 { t.Fatalf("alert without credentials: %d", code) } resp = c.req("POST", "/api/v1/alerts", map[string]string{"key": "cpu", "summary": "CPU high", "severity": "warning"}, bearer) json.NewDecoder(resp.Body).Decode(&out) resp.Body.Close() id, _ := out["id"].(string) if id == "" { t.Fatalf("no alert id: %v", out) } c.req("POST", "/api/v1/alerts/"+id+"/ack", nil, bearer).Body.Close() a, _ := s.getAlert(id) if a.Status != "acked" || a.AckedBy != "Root" { t.Fatalf("ack: %+v", a) } c.req("POST", "/api/v1/alerts", map[string]string{"key": "cpu", "status": "resolved"}, bearer).Body.Close() if a, _ = s.getAlert(id); a.Status != "resolved" { t.Fatalf("resolve via key: %+v", a) } } func TestAlertmanagerWebhook(t *testing.T) { s, _, _ := newTestServer(t) ts := httptest.NewServer(s.Handler()) defer ts.Close() c := newAPIClient(t, ts) c.setup() payload := map[string]any{"alerts": []map[string]any{{"status": "firing", "fingerprint": "abc", "labels": map[string]string{"alertname": "HighLatency", "severity": "critical"}, "annotations": map[string]string{"summary": "p99 > 2s"}}}} c.do("POST", "/api/v1/integrations/alertmanager", payload, nil) open := s.listAlerts("open", 10) if len(open) != 1 || open[0].Summary != "p99 > 2s" || open[0].Severity != "critical" { t.Fatalf("got %+v", open) } payload["alerts"].([]map[string]any)[0]["status"] = "resolved" c.do("POST", "/api/v1/integrations/alertmanager", payload, nil) if len(s.listAlerts("open", 10)) != 0 { t.Fatal("not resolved") } } func TestStatusPageOnlyShowsPublic(t *testing.T) { s, clk, _ := newTestServer(t) s.st.put("monitor", "a", &Monitor{ID: "a", Name: "Public site", Kind: "http", Target: "http://x", Enabled: true, Public: true}) s.st.put("monitor", "b", &Monitor{ID: "b", Name: "Internal DB", Kind: "tcp", Target: "x:1", Enabled: true}) s.saveState(MonitorState{MonitorID: "a", Status: "up", Since: clk.Now().Unix()}) s.st.put("incident", "i1", &Incident{ID: "i1", Title: "Public incident", Severity: "sev2", Status: "investigating", Public: true, Created: clk.Now().Unix()}) s.st.put("incident", "i2", &Incident{ID: "i2", Title: "Private incident", Severity: "sev1", Status: "investigating", Created: clk.Now().Unix()}) ts := httptest.NewServer(s.Handler()) defer ts.Close() resp, _ := http.Get(ts.URL + "/status.json") b, _ := io.ReadAll(resp.Body) body := string(b) if !strings.Contains(body, "Public site") || strings.Contains(body, "Internal DB") { t.Fatalf("monitor visibility wrong: %s", body) } if !strings.Contains(body, "Public incident") || strings.Contains(body, "Private incident") { t.Fatalf("incident visibility wrong: %s", body) } if !strings.Contains(body, `"overall":"degraded"`) { t.Fatalf("overall: %s", body) } } func TestLoginThrottle(t *testing.T) { s, _, _ := newTestServer(t) mustUser(t, s, "Alice") ts := httptest.NewServer(s.Handler()) defer ts.Close() c := newAPIClient(t, ts) var last int for i := 0; i < 12; i++ { last = c.do("POST", "/api/v1/login", map[string]string{"email": "alice@example.com", "password": "wrong-password-x"}, nil) } if last != 429 { t.Fatalf("expected throttling after repeated failures, got %d", last) } // Even the right password is refused while throttled. if code := c.do("POST", "/api/v1/login", map[string]string{"email": "alice@example.com", "password": "correct-horse-battery"}, nil); code != 429 { t.Fatalf("throttle should apply to the right password too, got %d", code) } }