Initial commit: notifications, monitoring, on-call and incidents
One binary and one SQLite file. Topics with ntfy-style publish and subscribe, HTTP/TCP/DNS/TLS/heartbeat monitors with a public status page, schedules and escalation policies, and incidents with a timeline and post-mortem. The UI follows the Lightwell design language. Builds against an unreleased Reef (secretbox) through a local replace line. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
VeillaxandClaude Sonnet 5.5 committed
commit
0c5a57556a
22 files changed
+4953
No files matched your search
@@ -0,0 +1,860 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Handler returns the HTTP handler for the whole app.
|
||||
func (s *Server) Handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// pages
|
||||
mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) { servePage(w, indexHTML) })
|
||||
mux.HandleFunc("GET /status", func(w http.ResponseWriter, r *http.Request) { servePage(w, statusHTML) })
|
||||
mux.HandleFunc("GET /status.json", s.statusJSON)
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "ok") })
|
||||
mux.HandleFunc("GET /metrics", s.metrics)
|
||||
mux.HandleFunc("GET /api/v1/hb/{token}", s.heartbeat)
|
||||
mux.HandleFunc("POST /api/v1/hb/{token}", s.heartbeat)
|
||||
|
||||
// session
|
||||
mux.HandleFunc("GET /api/v1/state", s.getState)
|
||||
mux.HandleFunc("POST /api/v1/setup", s.setup)
|
||||
mux.HandleFunc("POST /api/v1/login", s.apiLogin)
|
||||
mux.HandleFunc("POST /api/v1/logout", func(w http.ResponseWriter, r *http.Request) { s.logout(w, r); writeJSON(w, 200, ok) })
|
||||
|
||||
// topics, ntfy-style
|
||||
mux.HandleFunc("POST /{topic}", s.auth(false, s.publishHTTP))
|
||||
mux.HandleFunc("PUT /{topic}", s.auth(false, s.publishHTTP))
|
||||
mux.HandleFunc("GET /{topic}/sse", s.auth(false, s.subscribeHTTP))
|
||||
mux.HandleFunc("GET /{topic}/json", s.auth(false, s.subscribeHTTP))
|
||||
mux.HandleFunc("GET /api/v1/topics/{topic}/messages", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
|
||||
writeJSON(w, 200, s.MessagesSince(r.PathValue("topic"), since, 200))
|
||||
}))
|
||||
|
||||
// people
|
||||
mux.HandleFunc("GET /api/v1/users", s.auth(false, func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, s.listUsers()) }))
|
||||
mux.HandleFunc("POST /api/v1/users", s.auth(true, s.createUserHTTP))
|
||||
mux.HandleFunc("DELETE /api/v1/users/{id}", s.auth(true, s.deleteUser))
|
||||
mux.HandleFunc("POST /api/v1/users/{id}/password", s.auth(false, s.passwordHTTP))
|
||||
mux.HandleFunc("GET /api/v1/apikeys", s.auth(false, s.listKeys))
|
||||
mux.HandleFunc("POST /api/v1/apikeys", s.auth(false, s.createKeyHTTP))
|
||||
mux.HandleFunc("DELETE /api/v1/apikeys/{id}", s.auth(false, s.deleteKey))
|
||||
|
||||
// configuration documents
|
||||
s.crud(mux, "channels", "channel", true, func() any { return &Channel{} }, func(v any, id string) error {
|
||||
ch := v.(*Channel)
|
||||
ch.ID = id
|
||||
if ch.Target == redactedTarget { // an edit that left the hidden secret alone keeps the stored one
|
||||
old, err := getOf[Channel](s.st, "channel", id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ch.Target = s.st.open(old.Target)
|
||||
}
|
||||
if err := validateChannel(ch); err != nil {
|
||||
return err
|
||||
}
|
||||
ch.Target = s.st.seal(ch.Target)
|
||||
return nil
|
||||
}, redactChannel)
|
||||
s.crud(mux, "monitors", "monitor", true, func() any { return &Monitor{} }, func(v any, id string) error {
|
||||
m := v.(*Monitor)
|
||||
m.ID = id
|
||||
return validateMonitor(m)
|
||||
}, nil)
|
||||
s.crud(mux, "schedules", "schedule", true, func() any { return &Schedule{} }, func(v any, id string) error {
|
||||
sc := v.(*Schedule)
|
||||
sc.ID = id
|
||||
if strings.TrimSpace(sc.Name) == "" {
|
||||
return errors.New("give the schedule a name")
|
||||
}
|
||||
if sc.RotationHours < 1 {
|
||||
sc.RotationHours = 24 * 7
|
||||
}
|
||||
if sc.Start == 0 {
|
||||
sc.Start = s.now().Unix()
|
||||
}
|
||||
for _, o := range sc.Overrides {
|
||||
if o.To <= o.From {
|
||||
return errors.New("an override must end after it starts")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, nil)
|
||||
s.crud(mux, "policies", "policy", true, func() any { return &Policy{} }, func(v any, id string) error {
|
||||
p := v.(*Policy)
|
||||
p.ID = id
|
||||
if strings.TrimSpace(p.Name) == "" {
|
||||
return errors.New("give the policy a name")
|
||||
}
|
||||
if len(p.Steps) == 0 {
|
||||
return errors.New("a policy needs at least one step")
|
||||
}
|
||||
return nil
|
||||
}, nil)
|
||||
mux.HandleFunc("POST /api/v1/channels/{id}/test", s.auth(true, s.testChannel))
|
||||
mux.HandleFunc("GET /api/v1/oncall", s.auth(false, s.onCallNow))
|
||||
|
||||
// monitors
|
||||
mux.HandleFunc("GET /api/v1/monitors/{id}/checks", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 200, s.recentChecks(r.PathValue("id"), 100))
|
||||
}))
|
||||
mux.HandleFunc("GET /api/v1/monitor-states", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
out := map[string]MonitorState{}
|
||||
mons, _ := listOf[Monitor](s.st, "monitor")
|
||||
for _, m := range mons {
|
||||
out[m.ID] = s.monitorState(m.ID)
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}))
|
||||
|
||||
// alerts
|
||||
mux.HandleFunc("GET /api/v1/alerts", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
writeJSON(w, 200, s.listAlerts(r.URL.Query().Get("status"), limit))
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/alerts", s.auth(false, s.inboundAlert))
|
||||
mux.HandleFunc("POST /api/v1/integrations/alertmanager", s.auth(false, s.alertmanager))
|
||||
mux.HandleFunc("GET /api/v1/alerts/{id}", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
a, err := s.getAlert(r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
a.Timeline = s.st.events("alert", a.ID)
|
||||
writeJSON(w, 200, a)
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/alerts/{id}/ack", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
fail2(w, s.AckAlert(r.PathValue("id"), userFrom(r.Context()).Name))
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/alerts/{id}/resolve", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
fail2(w, s.ResolveAlert(r.PathValue("id"), userFrom(r.Context()).Name))
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/alerts/{id}/snooze", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Minutes int `json:"minutes"`
|
||||
}
|
||||
decode(r, &in)
|
||||
if in.Minutes < 1 || in.Minutes > 24*60 {
|
||||
in.Minutes = 30
|
||||
}
|
||||
fail2(w, s.SnoozeAlert(r.PathValue("id"), userFrom(r.Context()).Name, time.Duration(in.Minutes)*time.Minute))
|
||||
}))
|
||||
|
||||
// incidents
|
||||
mux.HandleFunc("GET /api/v1/incidents", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
list, err := listOf[Incident](s.st, "incident")
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
sortIncidents(list) // ids are random, so order by time
|
||||
writeJSON(w, 200, list)
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/incidents", s.auth(false, s.declareHTTP))
|
||||
mux.HandleFunc("GET /api/v1/incidents/{id}", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
in, err := getOf[Incident](s.st, "incident", r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
in.Timeline = s.st.events("incident", in.ID)
|
||||
writeJSON(w, 200, in)
|
||||
}))
|
||||
mux.HandleFunc("PATCH /api/v1/incidents/{id}", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
var p IncidentPatch
|
||||
if !decode(r, &p) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
in, err := s.UpdateIncident(r.PathValue("id"), userFrom(r.Context()).Name, p)
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, in)
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/incidents/{id}/comments", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
decode(r, &in)
|
||||
fail2(w, s.CommentIncident(r.PathValue("id"), userFrom(r.Context()).Name, in.Text))
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/incidents/{id}/postmortem-template", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
in, err := getOf[Incident](s.st, "incident", r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]string{"template": s.postmortemTemplate(in)})
|
||||
}))
|
||||
mux.HandleFunc("DELETE /api/v1/incidents/{id}", s.auth(true, func(w http.ResponseWriter, r *http.Request) {
|
||||
fail2(w, s.st.del("incident", r.PathValue("id")))
|
||||
}))
|
||||
|
||||
return secureHeaders(mux)
|
||||
}
|
||||
|
||||
var ok = map[string]bool{"ok": true}
|
||||
|
||||
func servePage(w http.ResponseWriter, page []byte) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.Write(page)
|
||||
}
|
||||
|
||||
func secureHeaders(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h := w.Header()
|
||||
h.Set("X-Content-Type-Options", "nosniff")
|
||||
h.Set("Referrer-Policy", "no-referrer")
|
||||
h.Set("X-Frame-Options", "DENY")
|
||||
h.Set("Content-Security-Policy", "default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func writeJSON(w http.ResponseWriter, code int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
// bad answers 400. Messages are capitalized here, in one place, so errors read as sentences.
|
||||
func bad(w http.ResponseWriter, msg string) {
|
||||
if msg != "" {
|
||||
r, n := utf8.DecodeRuneInString(msg)
|
||||
msg = string(unicode.ToUpper(r)) + msg[n:]
|
||||
}
|
||||
writeJSON(w, 400, map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
func fail(w http.ResponseWriter, err error) {
|
||||
if errors.Is(err, errNotFound) {
|
||||
writeJSON(w, 404, map[string]string{"error": "not found"})
|
||||
return
|
||||
}
|
||||
bad(w, err.Error())
|
||||
}
|
||||
|
||||
// fail2 answers with an error, or with ok when err is nil.
|
||||
func fail2(w http.ResponseWriter, err error) {
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, ok)
|
||||
}
|
||||
|
||||
func decode(r *http.Request, v any) bool {
|
||||
return json.NewDecoder(http.MaxBytesReader(nil, r.Body, 1<<20)).Decode(v) == nil
|
||||
}
|
||||
|
||||
// auth wraps a handler: it needs a signed-in user (admin if adminOnly). Requests
|
||||
// that change things with a cookie must be JSON, which a cross-site form can't send.
|
||||
func (s *Server) auth(adminOnly bool, h http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
u := s.authenticate(r)
|
||||
if u == nil {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="sandbar"`)
|
||||
writeJSON(w, 401, map[string]string{"error": "sign in or send an API key"})
|
||||
return
|
||||
}
|
||||
if adminOnly && u.Role != "admin" {
|
||||
writeJSON(w, 403, map[string]string{"error": "administrators only"})
|
||||
return
|
||||
}
|
||||
_, noCookie := r.Cookie(sessionCookie)
|
||||
if noCookie == nil && r.Method != "GET" && strings.HasPrefix(r.URL.Path, "/api/") &&
|
||||
!strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
bad(w, "send application/json")
|
||||
return
|
||||
}
|
||||
h(w, r.WithContext(context.WithValue(r.Context(), ctxKey{}, u)))
|
||||
}
|
||||
}
|
||||
|
||||
// crud registers list/create/update/delete for a kind of configuration document.
|
||||
func (s *Server) crud(mux *http.ServeMux, path, kind string, adminWrite bool, newItem func() any, prepare func(v any, id string) error, redact func(any) any) {
|
||||
base := "/api/v1/" + path
|
||||
read := func(v any) any {
|
||||
if redact != nil {
|
||||
return redact(v)
|
||||
}
|
||||
return v
|
||||
}
|
||||
mux.HandleFunc("GET "+base, s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
var out []any
|
||||
s.st.list(kind, func(raw []byte) error {
|
||||
v := newItem()
|
||||
if err := json.Unmarshal(raw, v); err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, read(v))
|
||||
return nil
|
||||
})
|
||||
if out == nil {
|
||||
out = []any{}
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}))
|
||||
save := func(w http.ResponseWriter, r *http.Request, id string) {
|
||||
v := newItem()
|
||||
if !decode(r, v) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
if err := prepare(v, id); err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
if err := s.st.put(kind, id, v); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, read(v))
|
||||
}
|
||||
mux.HandleFunc("POST "+base, s.auth(adminWrite, func(w http.ResponseWriter, r *http.Request) { save(w, r, newID()) }))
|
||||
mux.HandleFunc("PUT "+base+"/{id}", s.auth(adminWrite, func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if err := s.st.get(kind, id, newItem()); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
// A channel's secret target is never sent to the browser; an unchanged form sends the placeholder.
|
||||
save(w, r, id)
|
||||
}))
|
||||
mux.HandleFunc("DELETE "+base+"/{id}", s.auth(adminWrite, func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if kind == "monitor" {
|
||||
s.deleteMonitorData(id)
|
||||
}
|
||||
fail2(w, s.st.del(kind, id))
|
||||
}))
|
||||
}
|
||||
|
||||
func redactChannel(v any) any {
|
||||
c := *(v.(*Channel))
|
||||
c.Target = redactedTarget // webhook URLs and addresses are secrets; re-enter to change
|
||||
return c
|
||||
}
|
||||
|
||||
const redactedTarget = "••••••••"
|
||||
|
||||
// ---------- session ----------
|
||||
|
||||
func (s *Server) getState(w http.ResponseWriter, r *http.Request) {
|
||||
out := map[string]any{"needsSetup": s.countUsers() == 0, "user": nil, "url": s.opts.URL}
|
||||
if u := s.authenticate(r); u != nil {
|
||||
out["user"] = u
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) setup(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct{ Name, Email, Password string }
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
if s.countUsers() > 0 {
|
||||
writeJSON(w, 403, map[string]string{"error": "already set up"})
|
||||
return
|
||||
}
|
||||
if _, err := s.createUser(in.Name, in.Email, in.Password, "admin", ""); err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
u, err := s.login(w, r, in.Email, in.Password)
|
||||
if err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, u)
|
||||
}
|
||||
|
||||
func (s *Server) apiLogin(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct{ Email, Password string }
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
u, err := s.login(w, r, in.Email, in.Password)
|
||||
if err != nil {
|
||||
code := 401
|
||||
if errors.Is(err, errThrottled) {
|
||||
code = 429
|
||||
}
|
||||
writeJSON(w, code, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, u)
|
||||
}
|
||||
|
||||
// ---------- people ----------
|
||||
|
||||
func (s *Server) createUserHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct{ Name, Email, Password, Role, Phone string }
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
if in.Role == "" {
|
||||
in.Role = "member"
|
||||
}
|
||||
u, err := s.createUser(in.Name, in.Email, in.Password, in.Role, in.Phone)
|
||||
if err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, u)
|
||||
}
|
||||
|
||||
func (s *Server) deleteUser(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if id == userFrom(r.Context()).ID {
|
||||
bad(w, "you can't delete yourself")
|
||||
return
|
||||
}
|
||||
res, err := s.st.db.Exec(`DELETE FROM users WHERE id=?`, id)
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
fail(w, errNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, ok)
|
||||
}
|
||||
|
||||
func (s *Server) passwordHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
me := userFrom(r.Context())
|
||||
id := r.PathValue("id")
|
||||
if id != me.ID && me.Role != "admin" {
|
||||
writeJSON(w, 403, map[string]string{"error": "you can only change your own password"})
|
||||
return
|
||||
}
|
||||
var in struct{ Password string }
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
fail2(w, s.setPassword(id, in.Password))
|
||||
}
|
||||
|
||||
func (s *Server) listKeys(w http.ResponseWriter, r *http.Request) {
|
||||
out := []map[string]any{}
|
||||
rows, err := s.st.db.Query(`SELECT id,name,created FROM apikeys WHERE user_id=? ORDER BY created`, userFrom(r.Context()).ID)
|
||||
if err == nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id, name string
|
||||
var c int64
|
||||
if rows.Scan(&id, &name, &c) == nil {
|
||||
out = append(out, map[string]any{"id": id, "name": name, "created": c})
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) createKeyHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct{ Name string }
|
||||
decode(r, &in)
|
||||
id, tok, err := s.createAPIKey(userFrom(r.Context()).ID, in.Name)
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]string{"id": id, "token": tok}) // shown once
|
||||
}
|
||||
|
||||
func (s *Server) deleteKey(w http.ResponseWriter, r *http.Request) {
|
||||
s.st.db.Exec(`DELETE FROM apikeys WHERE id=? AND user_id=?`, r.PathValue("id"), userFrom(r.Context()).ID)
|
||||
writeJSON(w, 200, ok)
|
||||
}
|
||||
|
||||
// ---------- topics ----------
|
||||
|
||||
func (s *Server) publishHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
topic := r.PathValue("topic")
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
|
||||
if err != nil {
|
||||
bad(w, "message too large (limit 64 KB)")
|
||||
return
|
||||
}
|
||||
h := func(names ...string) string {
|
||||
for _, n := range names {
|
||||
if v := r.Header.Get(n); v != "" {
|
||||
return v
|
||||
}
|
||||
if v := r.URL.Query().Get(strings.ToLower(n)); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
m, err := s.Publish(topic, h("Title", "X-Title", "t"), string(body), parsePriority(h("Priority", "X-Priority", "p")), parseTags(h("Tags", "X-Tags", "ta")))
|
||||
if err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, m)
|
||||
}
|
||||
|
||||
// subscribeHTTP streams a topic as server-sent events (/sse) or JSON lines (/json).
|
||||
// ?since=<unix seconds>|all replays stored messages first.
|
||||
func (s *Server) subscribeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
topic := r.PathValue("topic")
|
||||
if !validTopic(topic) {
|
||||
bad(w, "invalid topic")
|
||||
return
|
||||
}
|
||||
fl, canFlush := w.(http.Flusher)
|
||||
if !canFlush {
|
||||
http.Error(w, "streaming not supported", 500)
|
||||
return
|
||||
}
|
||||
sse := strings.HasSuffix(r.URL.Path, "/sse")
|
||||
if sse {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "application/x-ndjson")
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("X-Accel-Buffering", "no")
|
||||
bw := bufio.NewWriter(w)
|
||||
write := func(m Message) bool {
|
||||
b, _ := json.Marshal(m)
|
||||
var err error
|
||||
if sse {
|
||||
_, err = fmt.Fprintf(bw, "id: %s\ndata: %s\n\n", m.ID, b)
|
||||
} else {
|
||||
_, err = fmt.Fprintf(bw, "%s\n", b)
|
||||
}
|
||||
bw.Flush()
|
||||
fl.Flush()
|
||||
return err == nil
|
||||
}
|
||||
// Subscribe before replaying so nothing published in between is missed; duplicates are skipped by id.
|
||||
ch, unsub := s.hub.subscribe(topic)
|
||||
defer unsub()
|
||||
seen := map[string]bool{}
|
||||
if q := r.URL.Query().Get("since"); q != "" {
|
||||
since := int64(0)
|
||||
if q != "all" {
|
||||
since, _ = strconv.ParseInt(q, 10, 64)
|
||||
}
|
||||
for _, m := range s.MessagesSince(topic, since, 500) {
|
||||
seen[m.ID] = true
|
||||
if !write(m) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if sse {
|
||||
fmt.Fprint(bw, ": connected\n\n")
|
||||
bw.Flush()
|
||||
fl.Flush()
|
||||
}
|
||||
keepalive := time.NewTicker(30 * time.Second)
|
||||
defer keepalive.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-s.stop:
|
||||
return
|
||||
case m := <-ch:
|
||||
if seen[m.ID] {
|
||||
continue
|
||||
}
|
||||
if !write(m) {
|
||||
return
|
||||
}
|
||||
case <-keepalive.C:
|
||||
if sse {
|
||||
fmt.Fprint(bw, ": keepalive\n\n")
|
||||
} else {
|
||||
fmt.Fprint(bw, "\n")
|
||||
}
|
||||
bw.Flush()
|
||||
fl.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- alerts in ----------
|
||||
|
||||
func (s *Server) inboundAlert(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Key string `json:"key"`
|
||||
Summary string `json:"summary"`
|
||||
Detail string `json:"detail"`
|
||||
Severity string `json:"severity"`
|
||||
Policy string `json:"policy"`
|
||||
Status string `json:"status"` // firing (default) or resolved
|
||||
Source string `json:"source"`
|
||||
}
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
if in.Key == "" {
|
||||
in.Key = in.Summary
|
||||
}
|
||||
if in.Key == "" {
|
||||
bad(w, "send a key or a summary")
|
||||
return
|
||||
}
|
||||
if in.Source == "" {
|
||||
in.Source = "api"
|
||||
}
|
||||
in.Key = "ext:" + in.Key
|
||||
if in.Status == "resolved" {
|
||||
s.ResolveAlertKey(in.Key, in.Source)
|
||||
writeJSON(w, 200, ok)
|
||||
return
|
||||
}
|
||||
if in.Summary == "" {
|
||||
in.Summary = in.Key
|
||||
}
|
||||
if in.Policy != "" {
|
||||
if _, err := getOf[Policy](s.st, "policy", in.Policy); err != nil {
|
||||
bad(w, "unknown escalation policy")
|
||||
return
|
||||
}
|
||||
}
|
||||
a, isNew, err := s.RaiseAlert(in.Key, in.Source, in.Summary, in.Detail, in.Severity, in.Policy)
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"id": a.ID, "new": isNew})
|
||||
}
|
||||
|
||||
// alertmanager accepts Prometheus Alertmanager's webhook (also Grafana's, which uses the same shape).
|
||||
// ?policy=<id> sets the escalation policy.
|
||||
func (s *Server) alertmanager(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Alerts []struct {
|
||||
Status string `json:"status"`
|
||||
Labels map[string]string `json:"labels"`
|
||||
Annotations map[string]string `json:"annotations"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
} `json:"alerts"`
|
||||
}
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
policy := r.URL.Query().Get("policy")
|
||||
if policy != "" {
|
||||
if _, err := getOf[Policy](s.st, "policy", policy); err != nil {
|
||||
bad(w, "unknown escalation policy")
|
||||
return
|
||||
}
|
||||
}
|
||||
n := 0
|
||||
for _, a := range in.Alerts {
|
||||
key := a.Fingerprint
|
||||
if key == "" {
|
||||
key = a.Labels["alertname"] + "/" + a.Labels["instance"]
|
||||
}
|
||||
key = "am:" + key
|
||||
if a.Status == "resolved" {
|
||||
s.ResolveAlertKey(key, "alertmanager")
|
||||
continue
|
||||
}
|
||||
sev := map[string]string{"critical": "critical", "warning": "warning", "info": "info"}[a.Labels["severity"]]
|
||||
summary := firstNonEmpty(a.Annotations["summary"], a.Labels["alertname"], key)
|
||||
if _, _, err := s.RaiseAlert(key, "alertmanager", summary, a.Annotations["description"], sev, policy); err == nil {
|
||||
n++
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, map[string]int{"processed": n})
|
||||
}
|
||||
|
||||
// ---------- misc ----------
|
||||
|
||||
func (s *Server) declareHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Title string `json:"title"`
|
||||
Severity string `json:"severity"`
|
||||
Summary string `json:"summary"`
|
||||
Lead string `json:"lead"`
|
||||
AlertIDs []string `json:"alertIds"`
|
||||
}
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
me := userFrom(r.Context())
|
||||
if in.Lead == "" {
|
||||
in.Lead = me.ID
|
||||
}
|
||||
inc, err := s.DeclareIncident(in.Title, in.Severity, in.Summary, in.Lead, me.Name, in.AlertIDs)
|
||||
if err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, inc)
|
||||
}
|
||||
|
||||
func (s *Server) testChannel(w http.ResponseWriter, r *http.Request) {
|
||||
ch, err := getOf[Channel](s.st, "channel", r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
if err := s.sendChannel(*ch, Note{Status: "test", Kind: "alert", Title: "test", Time: s.now().Unix()}); err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, ok)
|
||||
}
|
||||
|
||||
func (s *Server) onCallNow(w http.ResponseWriter, r *http.Request) {
|
||||
type row struct {
|
||||
Schedule string `json:"schedule"`
|
||||
ID string `json:"id"`
|
||||
UserID string `json:"userId"`
|
||||
User string `json:"user"`
|
||||
Next string `json:"next"` // who is on call after the current handover
|
||||
}
|
||||
out := []row{}
|
||||
scs, _ := listOf[Schedule](s.st, "schedule")
|
||||
now := s.now()
|
||||
for _, sc := range scs {
|
||||
sc := sc
|
||||
uid := sc.OnCall(now)
|
||||
nx := sc.OnCall(now.Add(time.Duration(sc.RotationHours) * time.Hour))
|
||||
out = append(out, row{sc.Name, sc.ID, uid, s.userName(uid), s.userName(nx)})
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) heartbeat(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.Heartbeat(r.PathValue("token")) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
io.WriteString(w, "ok")
|
||||
}
|
||||
|
||||
// statusJSON feeds the public status page: only monitors and incidents marked public.
|
||||
func (s *Server) statusJSON(w http.ResponseWriter, r *http.Request) {
|
||||
type mon struct {
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Uptime float64 `json:"uptime"`
|
||||
Since int64 `json:"since"`
|
||||
}
|
||||
type inc struct {
|
||||
Title string `json:"title"`
|
||||
Status string `json:"status"`
|
||||
Severity string `json:"severity"`
|
||||
Created int64 `json:"created"`
|
||||
Resolved int64 `json:"resolved"`
|
||||
Updates []Event `json:"updates"`
|
||||
}
|
||||
out := struct {
|
||||
Monitors []mon `json:"monitors"`
|
||||
Incidents []inc `json:"incidents"`
|
||||
Overall string `json:"overall"`
|
||||
}{[]mon{}, []inc{}, "operational"}
|
||||
mons, _ := listOf[Monitor](s.st, "monitor")
|
||||
for _, m := range mons {
|
||||
if !m.Public || !m.Enabled {
|
||||
continue
|
||||
}
|
||||
st := s.monitorState(m.ID)
|
||||
out.Monitors = append(out.Monitors, mon{m.Name, st.Status, st.Uptime30, st.Since})
|
||||
if st.Status == "down" {
|
||||
out.Overall = "outage"
|
||||
}
|
||||
}
|
||||
incs, _ := listOf[Incident](s.st, "incident")
|
||||
sortIncidents(incs)
|
||||
cut := s.now().AddDate(0, 0, -14).Unix()
|
||||
for _, in := range incs {
|
||||
if !in.Public || (in.Status == "resolved" && in.Resolved < cut) {
|
||||
continue
|
||||
}
|
||||
var ups []Event
|
||||
for _, e := range s.st.events("incident", in.ID) {
|
||||
if e.Kind == "update" || e.Kind == "declared" {
|
||||
e.Actor = "" // staff names stay off the public page
|
||||
ups = append(ups, e)
|
||||
}
|
||||
}
|
||||
out.Incidents = append(out.Incidents, inc{in.Title, in.Status, in.Severity, in.Created, in.Resolved, ups})
|
||||
if in.Status != "resolved" && out.Overall == "operational" {
|
||||
out.Overall = "degraded"
|
||||
}
|
||||
}
|
||||
w.Header().Set("Cache-Control", "max-age=15")
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
|
||||
func sortIncidents(list []Incident) {
|
||||
for i := 1; i < len(list); i++ { // newest first; lists are short
|
||||
for j := i; j > 0 && list[j].Created > list[j-1].Created; j-- {
|
||||
list[j], list[j-1] = list[j-1], list[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) metrics(w http.ResponseWriter, r *http.Request) {
|
||||
if s.opts.MetricsToken == "" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if h := r.Header.Get("Authorization"); h != "Bearer "+s.opts.MetricsToken {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="metrics"`)
|
||||
http.Error(w, "unauthorized", 401)
|
||||
return
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("# TYPE sandbar_monitor_up gauge\n")
|
||||
mons, _ := listOf[Monitor](s.st, "monitor")
|
||||
for _, m := range mons {
|
||||
v := 0
|
||||
if s.monitorState(m.ID).Status == "up" {
|
||||
v = 1
|
||||
}
|
||||
fmt.Fprintf(&b, "sandbar_monitor_up{monitor=%q} %d\n", m.Name, v)
|
||||
}
|
||||
var firing, acked, open int
|
||||
s.st.db.QueryRow(`SELECT COUNT(*) FROM alerts WHERE status='firing'`).Scan(&firing)
|
||||
s.st.db.QueryRow(`SELECT COUNT(*) FROM alerts WHERE status='acked'`).Scan(&acked)
|
||||
incs, _ := listOf[Incident](s.st, "incident")
|
||||
for _, in := range incs {
|
||||
if in.Status != "resolved" {
|
||||
open++
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "# TYPE sandbar_alerts gauge\nsandbar_alerts{status=\"firing\"} %d\nsandbar_alerts{status=\"acked\"} %d\n", firing, acked)
|
||||
fmt.Fprintf(&b, "# TYPE sandbar_incidents_open gauge\nsandbar_incidents_open %d\n", open)
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
|
||||
io.WriteString(w, b.String())
|
||||
}
|
||||
Reference in new issue
Block a user